ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

staggered skipframes

pull/50/head
Vladimir Mandic 2020-12-10 14:47:53 -05:00
parent 71203f08ee
commit 3b5cc007c0
16 changed files with 113 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
config.js
View File

@ -66,9 +66,10 @@ export default {
// 'back' is optimized for distanct faces.
inputSize: 256, // fixed value: 128 for front and 256 for 'back'
rotation: false, // use best-guess rotated face image or just box with rotation as-is
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
maxFaces: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance
skipFrames: 15, // how many frames to go without re-running the face bounding box detector
skipFrames: 20, // how many frames to go without re-running the face bounding box detector
// only used for video inputs
// e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated face analysis as the head probably hasn't moved much
@ -98,7 +99,7 @@ export default {
modelPath: '../models/age-ssrnet-imdb.json', // can be 'age-ssrnet-imdb' or 'age-ssrnet-wiki'
// which determines training set for model
inputSize: 64, // fixed value
skipFrames: 15, // how many frames to go without re-running the detector
skipFrames: 41, // how many frames to go without re-running the detector
// only used for video inputs
},
@ -107,7 +108,7 @@ export default {
minConfidence: 0.1, // threshold for discarding a prediction
modelPath: '../models/gender-ssrnet-imdb.json', // can be 'gender', 'gender-ssrnet-imdb' or 'gender-ssrnet-wiki'
inputSize: 64, // fixed value
skipFrames: 15, // how many frames to go without re-running the detector
skipFrames: 42, // how many frames to go without re-running the detector
// only used for video inputs
},
@ -115,7 +116,7 @@ export default {
enabled: true,
inputSize: 64, // fixed value
minConfidence: 0.2, // threshold for discarding a prediction
skipFrames: 15, // how many frames to go without re-running the detector
skipFrames: 21, // how many frames to go without re-running the detector
modelPath: '../models/emotion-large.json', // can be 'mini', 'large'
},
@ -140,15 +141,15 @@ export default {
hand: {
enabled: true,
inputSize: 256, // fixed value
skipFrames: 15, // how many frames to go without re-running the hand bounding box detector
skipFrames: 19, // how many frames to go without re-running the hand bounding box detector
// only used for video inputs
// e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated hand skeleton analysis as the hand probably
// hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction
minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much
// in non-maximum suppression
scoreThreshold: 0.8, // threshold for deciding when to remove boxes based on
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on
// score in non-maximum suppression
maxHands: 1, // maximum number of hands detected in the input
// should be set to the minimum number for performance

10
demo/browser.js
View File

@ -3,7 +3,10 @@ import draw from './draw.js';
import Menu from './menu.js';
import GLBench from './gl-bench.js';
const userConfig = {}; // add any user configuration overrides
// const userConfig = {}; // add any user configuration overrides
const userConfig = {
async: false,
};
const human = new Human(userConfig);
@ -30,7 +33,7 @@ const ui = {
console: true,
maxFPSframes: 10,
modelsPreload: true,
modelsWarmup: true,
modelsWarmup: false,
menuWidth: 0,
menuHeight: 0,
camera: {},
@ -41,7 +44,7 @@ const ui = {
detectThread: null,
framesDraw: 0,
framesDetect: 0,
bench: false,
bench: true,
};
// global variables
@ -283,6 +286,7 @@ function runHumanDetect(input, canvas, timestamp) {
clearTimeout(ui.drawThread);
ui.drawThread = null;
log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw);
log('memory', human.tf.engine().memory());
return;
}
status('');

10
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

12
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"dist/human.esm.js": {
"bytes": 1786694,
"bytes": 1786626,
"imports": []
},
"demo/draw.js": {
@ -17,7 +17,7 @@
"imports": []
},
"demo/browser.js": {
"bytes": 25133,
"bytes": 25223,
"imports": [
{
"path": "dist/human.esm.js"
@ -38,14 +38,14 @@
"dist/demo-browser-index.js.map": {
"imports": [],
"inputs": {},
"bytes": 2746528
"bytes": 2746890
},
"dist/demo-browser-index.js": {
"imports": [],
"exports": [],
"inputs": {
"dist/human.esm.js": {
"bytesInOutput": 1779454
"bytesInOutput": 1779386
},
"demo/draw.js": {
"bytesInOutput": 7816
@ -57,10 +57,10 @@
"bytesInOutput": 7382
},
"demo/browser.js": {
"bytesInOutput": 19361
"bytesInOutput": 19411
}
},
"bytes": 1833202
"bytes": 1833184
}
}
}

6
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

19
dist/human.esm.json vendored
View File

@ -28,7 +28,7 @@
]
},
"src/face/util.js": {
"bytes": 3078,
"bytes": 3087,
"imports": []
},
"src/face/coords.js": {
@ -36,7 +36,7 @@
"imports": []
},
"src/face/facepipeline.js": {
"bytes": 13876,
"bytes": 13817,
"imports": [
{
"path": "dist/tfjs.esm.js"
@ -49,11 +49,14 @@
},
{
"path": "src/face/coords.js"
},
{
"path": "src/log.js"
}
]
},
"src/face/facemesh.js": {
"bytes": 2798,
"bytes": 2816,
"imports": [
{
"path": "src/log.js"
@ -336,7 +339,7 @@
]
},
"config.js": {
"bytes": 8654,
"bytes": 8779,
"imports": []
},
"src/sample.js": {
@ -402,7 +405,7 @@
"dist/human.esm.js.map": {
"imports": [],
"inputs": {},
"bytes": 2651413
"bytes": 2651614
},
"dist/human.esm.js": {
"imports": [],
@ -423,10 +426,10 @@
"bytesInOutput": 30817
},
"src/face/facepipeline.js": {
"bytesInOutput": 9389
"bytesInOutput": 9297
},
"src/face/facemesh.js": {
"bytesInOutput": 2158
"bytesInOutput": 2182
},
"src/profile.js": {
"bytesInOutput": 846
@ -522,7 +525,7 @@
"bytesInOutput": 21
}
},
"bytes": 1786694
"bytes": 1786626
}
}
}

6
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

19
dist/human.json vendored
View File

@ -28,7 +28,7 @@
]
},
"src/face/util.js": {
"bytes": 3078,
"bytes": 3087,
"imports": []
},
"src/face/coords.js": {
@ -36,7 +36,7 @@
"imports": []
},
"src/face/facepipeline.js": {
"bytes": 13876,
"bytes": 13817,
"imports": [
{
"path": "dist/tfjs.esm.js"
@ -49,11 +49,14 @@
},
{
"path": "src/face/coords.js"
},
{
"path": "src/log.js"
}
]
},
"src/face/facemesh.js": {
"bytes": 2798,
"bytes": 2816,
"imports": [
{
"path": "src/log.js"
@ -336,7 +339,7 @@
]
},
"config.js": {
"bytes": 8654,
"bytes": 8779,
"imports": []
},
"src/sample.js": {
@ -402,7 +405,7 @@
"dist/human.js.map": {
"imports": [],
"inputs": {},
"bytes": 2668410
"bytes": 2668611
},
"dist/human.js": {
"imports": [],
@ -421,10 +424,10 @@
"bytesInOutput": 30817
},
"src/face/facepipeline.js": {
"bytesInOutput": 9389
"bytesInOutput": 9297
},
"src/face/facemesh.js": {
"bytesInOutput": 2158
"bytesInOutput": 2182
},
"src/profile.js": {
"bytesInOutput": 846
@ -520,7 +523,7 @@
"bytesInOutput": 21
}
},
"bytes": 1786768
"bytes": 1786700
}
}
}

18
package-lock.json generated
View File

@ -323,9 +323,9 @@
"dev": true
},
"@types/node": {
"version": "14.14.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.10.tgz",
"integrity": "sha512-J32dgx2hw8vXrSbu4ZlVhn1Nm3GbeCFNw2FWL8S5QKucHGY0cyNwjdQdO+KMBZ4wpmC7KhLCiNsdk1RFRIYUQQ==",
"version": "14.14.12",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.12.tgz",
"integrity": "sha512-ASH8OPHMNlkdjrEdmoILmzFfsJICvhBsFfAum4aKZ/9U4B6M6tTmTPh+f3ttWdD74CEGV5XvXWkbyfSdXaTd7g==",
"dev": true
},
"@types/node-fetch": {
@ -1485,9 +1485,9 @@
"dev": true
},
"ini": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
"version": "1.3.7",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
"integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
"dev": true
},
"is-arrayish": {
@ -2258,9 +2258,9 @@
"dev": true
},
"simple-git": {
"version": "2.25.0",
"resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.25.0.tgz",
"integrity": "sha512-RMBFWKiPDl3rAoFbEaCVsjQ0v6sgR0q7cyUF9e/4lR81Mf2/5xwop0aCQatUDXKMIXnkuOG6aTEadQmGtOw4mg==",
"version": "2.26.0",
"resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.26.0.tgz",
"integrity": "sha512-I9QIjBNA773X23SZ/S1HFMCA+S//san83Twyd5ffWFjo/sv8VRk7tuck23y1uFWuzTu4KTwDh5LEXyCfEfOWMw==",
"dev": true,
"requires": {
"@kwsites/file-exists": "^1.1.1",

2
package.json
View File

@ -44,7 +44,7 @@
"eslint-plugin-promise": "^4.2.1",
"rimraf": "^3.0.2",
"seedrandom": "^3.0.5",
"simple-git": "^2.25.0"
"simple-git": "^2.26.0"
},
"scripts": {
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",

9
src/face/facemesh.js
View File

@ -1,21 +1,20 @@
import { log } from '../log.js';
import * as tf from '../../dist/tfjs.esm.js';
import * as blazeface from './blazeface.js';
import * as pipe from './facepipeline.js';
import * as facepipeline from './facepipeline.js';
import * as coords from './coords.js';
class MediaPipeFaceMesh {
constructor(blazeFace, blazeMeshModel, irisModel, config) {
this.pipeline = new pipe.Pipeline(blazeFace, blazeMeshModel, irisModel, config);
this.facePipeline = new facepipeline.Pipeline(blazeFace, blazeMeshModel, irisModel, config);
this.config = config;
}
async estimateFaces(input, config) {
const predictions = await this.pipeline.predict(input, config);
const predictions = await this.facePipeline.predict(input, config);
const results = [];
for (const prediction of (predictions || [])) {
// guard against disposed tensors on long running operations such as pause in middle of processing
if (prediction.isDisposedInternal) continue;
if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
const mesh = prediction.coords ? prediction.coords.arraySync() : null;
const annotations = {};
if (mesh && mesh.length > 0) {

67
src/face/facepipeline.js
View File

@ -3,6 +3,8 @@ import * as tf from '../../dist/tfjs.esm.js';
import * as bounding from './box';
import * as util from './util';
import * as coords from './coords.js';
// eslint-disable-next-line no-unused-vars
import { log } from '../log.js';
const LANDMARKS_COUNT = 468;
const MESH_MOUTH_INDEX = 13;
@ -60,17 +62,14 @@ class Pipeline {
scaleFactor[0] * (coord[0] - this.meshWidth / 2),
scaleFactor[1] * (coord[1] - this.meshHeight / 2), coord[2],
]));
const coordsRotationMatrix = util.buildRotationMatrix(angle, [0, 0]);
const coordsRotated = coordsScaled.map((coord) => ([...util.rotatePoint(coord, coordsRotationMatrix), coord[2]]));
const inverseRotationMatrix = util.invertTransformMatrix(rotationMatrix);
const coordsRotationMatrix = (angle !== 0) ? util.buildRotationMatrix(angle, [0, 0]) : util.IDENTITY_MATRIX;
const coordsRotated = (angle !== 0) ? coordsScaled.map((coord) => ([...util.rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled;
const inverseRotationMatrix = (angle !== 0) ? util.invertTransformMatrix(rotationMatrix) : util.IDENTITY_MATRIX;
const boxCenter = [...bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1];
const originalBoxCenter = [
util.dot(boxCenter, inverseRotationMatrix[0]),
util.dot(boxCenter, inverseRotationMatrix[1]),
];
return coordsRotated.map((coord) => ([
coord[0] + originalBoxCenter[0],
coord[1] + originalBoxCenter[1], coord[2],
coord[0] + util.dot(boxCenter, inverseRotationMatrix[0]),
coord[1] + util.dot(boxCenter, inverseRotationMatrix[1]),
coord[2],
]));
}
@ -174,26 +173,23 @@ class Pipeline {
}
// log(this.skipped, config.face.detector.skipFrames, this.detectedFaces, config.face.detector.maxFaces, detector?.boxes.length, this.storedBoxes.length);
let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
let angle = 0;
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
const boxLandmarksFromMeshModel = box.landmarks.length >= LANDMARKS_COUNT;
let [indexOfMouth, indexOfForehead] = MESH_KEYPOINTS_LINE_OF_SYMMETRY_INDICES;
if (boxLandmarksFromMeshModel === false) {
[indexOfMouth, indexOfForehead] = BLAZEFACE_KEYPOINTS_LINE_OF_SYMMETRY_INDICES;
}
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
let rotatedImage = input;
let rotationMatrix = util.IDENTITY_MATRIX;
if (angle !== 0) {
rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
let face;
let angle = 0;
let rotationMatrix;
if (config.face.detector.rotation) {
const [indexOfMouth, indexOfForehead] = (box.landmarks.length >= LANDMARKS_COUNT) ? MESH_KEYPOINTS_LINE_OF_SYMMETRY_INDICES : BLAZEFACE_KEYPOINTS_LINE_OF_SYMMETRY_INDICES;
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshHeight, this.meshWidth]).div(255);
} else {
rotationMatrix = util.IDENTITY_MATRIX;
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, input, [this.meshHeight, this.meshWidth]).div(255);
}
const face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshHeight, this.meshWidth]).div(255);
const outputFace = config.face.detector.rotation ? tf.image.rotateWithOffset(face, angle) : face;
// if we're not going to produce mesh, don't spend time with further processing
if (!config.face.mesh.enabled) {
@ -202,27 +198,23 @@ class Pipeline {
box,
faceConfidence: null,
confidence: box.confidence,
image: outputFace,
image: face,
};
return prediction;
}
// The first returned tensor represents facial contours, which are included in the coordinates.
const [, confidence, contourCoords] = this.meshDetector.predict(face);
const [, confidence, contourCoords] = this.meshDetector.predict(face); // The first returned tensor represents facial contours, which are included in the coordinates.
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.face.detector.minConfidence) {
contourCoords.dispose();
return null;
}
if (confidenceVal < config.face.detector.minConfidence) return null; // if below confidence just exit
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
let rawCoords = coordsReshaped.arraySync();
if (config.face.iris.enabled) {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face, LEFT_EYE_BOUNDS[0], LEFT_EYE_BOUNDS[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face, RIGHT_EYE_BOUNDS[0], RIGHT_EYE_BOUNDS[1]);
const eyePredictions = (this.irisModel.predict(tf.concat([leftEyeCrop, rightEyeCrop])));
const eyePredictions = this.irisModel.predict(tf.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync();
eyePredictions.dispose();
const leftEyeData = eyePredictionsData.slice(0, IRIS_NUM_COORDINATES * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(IRIS_NUM_COORDINATES * 3);
@ -241,8 +233,8 @@ class Pipeline {
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, 'right');
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
}
const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
tf.dispose(rawCoords);
const landmarksBox = bounding.enlargeBox(this.calculateLandmarksBoundingBox(transformedCoordsData));
const transformedCoords = tf.tensor2d(transformedCoordsData);
const prediction = {
@ -250,9 +242,10 @@ class Pipeline {
box: landmarksBox,
faceConfidence: confidenceVal,
confidence: box.confidence,
image: outputFace,
image: face,
};
this.storedBoxes[i] = { ...landmarksBox, landmarks: transformedCoords.arraySync(), confidence: box.confidence, faceConfidence: confidenceVal };
return prediction;
}));
results = results.filter((a) => a !== null);

9
src/face/util.js
View File

@ -7,6 +7,7 @@ function normalizeRadians(angle) {
return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI));
}
exports.normalizeRadians = normalizeRadians;
/**
* Computes the angle of rotation between two anchor points.
* @param point1 First anchor point
@ -17,13 +18,16 @@ function computeRotation(point1, point2) {
return normalizeRadians(radians);
}
exports.computeRotation = computeRotation;
function radToDegrees(rad) {
return rad * 180 / Math.PI;
}
exports.radToDegrees = radToDegrees;
function buildTranslationMatrix(x, y) {
return [[1, 0, x], [0, 1, y], [0, 0, 1]];
}
function dot(v1, v2) {
let product = 0;
for (let i = 0; i < v1.length; i++) {
@ -32,6 +36,7 @@ function dot(v1, v2) {
return product;
}
exports.dot = dot;
function getColumnFrom2DArr(arr, columnIndex) {
const column = [];
for (let i = 0; i < arr.length; i++) {
@ -40,6 +45,7 @@ function getColumnFrom2DArr(arr, columnIndex) {
return column;
}
exports.getColumnFrom2DArr = getColumnFrom2DArr;
function multiplyTransformMatrices(mat1, mat2) {
const product = [];
const size = mat1.length;
@ -61,6 +67,7 @@ function buildRotationMatrix(rotation, center) {
return multiplyTransformMatrices(translationTimesRotation, negativeTranslationMatrix);
}
exports.buildRotationMatrix = buildRotationMatrix;
function invertTransformMatrix(matrix) {
const rotationComponent = [[matrix[0][0], matrix[1][0]], [matrix[0][1], matrix[1][1]]];
const translationComponent = [matrix[0][2], matrix[1][2]];
@ -75,6 +82,7 @@ function invertTransformMatrix(matrix) {
];
}
exports.invertTransformMatrix = invertTransformMatrix;
function rotatePoint(homogeneousCoordinate, rotationMatrix) {
return [
dot(homogeneousCoordinate, rotationMatrix[0]),
@ -82,6 +90,7 @@ function rotatePoint(homogeneousCoordinate, rotationMatrix) {
];
}
exports.rotatePoint = rotatePoint;
function xyDistanceBetweenPoints(a, b) {
return Math.sqrt(((a[0] - b[0]) ** 2) + ((a[1] - b[1]) ** 2));
}