ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix performance issue when running with low confidence

pull/91/head
Vladimir Mandic 2021-03-08 10:06:34 -05:00
parent 578195f472
commit 3588007f1c
24 changed files with 4958 additions and 1096 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.js
View File

@ -75,7 +75,7 @@ export default {
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees // false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
maxFaces: 10, // maximum number of faces detected in the input maxFaces: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
skipFrames: 11, // how many frames to go without re-running the face bounding box detector skipFrames: 21, // how many frames to go without re-running the face bounding box detector
// only used for video inputs // only used for video inputs
// e.g., if model is running st 25 FPS, we can re-use existing bounding // e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated face analysis as the head probably hasn't moved much // box for updated face analysis as the head probably hasn't moved much
@ -94,7 +94,6 @@ export default {
enabled: true, enabled: true,
modelPath: '../models/facemesh.json', modelPath: '../models/facemesh.json',
inputSize: 192, // fixed value inputSize: 192, // fixed value
returnRawData: false, // in addition to standard mesh and box values, return raw normalized values as well
}, },
iris: { iris: {
@ -116,7 +115,7 @@ export default {
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
modelPath: '../models/gender.json', // can be 'gender' or 'gender-ssrnet-imdb' modelPath: '../models/gender.json', // can be 'gender' or 'gender-ssrnet-imdb'
inputSize: 64, // fixed value inputSize: 64, // fixed value
skipFrames: 41, // how many frames to go without re-running the detector skipFrames: 32, // how many frames to go without re-running the detector
// only used for video inputs // only used for video inputs
}, },
@ -124,7 +123,7 @@ export default {
enabled: true, enabled: true,
inputSize: 64, // fixed value inputSize: 64, // fixed value
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 21, // how many frames to go without re-running the detector skipFrames: 33, // how many frames to go without re-running the detector
modelPath: '../models/emotion.json', modelPath: '../models/emotion.json',
}, },

7
demo/browser.js
View File

@ -6,13 +6,13 @@ const userConfig = { backend: 'webgl' }; // add any user configuration overrides
/* /*
const userConfig = { const userConfig = {
backend: 'wasm', backend: 'webgl',
async: false, async: false,
warmup: 'face', warmup: 'face',
videoOptimized: false, videoOptimized: true,
face: { enabled: true, iris: { enabled: false }, mesh: { enabled: true }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } }, face: { enabled: true, iris: { enabled: false }, mesh: { enabled: true }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
hand: { enabled: false }, hand: { enabled: false },
gestures: { enabled: true }, gesture: { enabled: false },
body: { enabled: false, modelType: 'blazepose', modelPath: '../models/blazepose.json' }, body: { enabled: false, modelType: 'blazepose', modelPath: '../models/blazepose.json' },
}; };
*/ */
@ -424,6 +424,7 @@ function setupMenu() {
}); });
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth'); menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('draw with curves', human.draw.options, 'useCurves');
menu.display.addBool('print labels', human.draw.options, 'drawLabels'); menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints'); menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes'); menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');

402
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

12
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1353240, "bytes": 1353178,
"imports": [] "imports": []
}, },
"demo/menu.js": { "demo/menu.js": {
@ -13,7 +13,7 @@
"imports": [] "imports": []
}, },
"demo/browser.js": { "demo/browser.js": {
"bytes": 27966, "bytes": 28043,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js", "path": "dist/human.esm.js",
@ -35,7 +35,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 2061718 "bytes": 2061556
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
@ -43,7 +43,7 @@
"entryPoint": "demo/browser.js", "entryPoint": "demo/browser.js",
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1345744 "bytesInOutput": 1345684
}, },
"demo/menu.js": { "demo/menu.js": {
"bytesInOutput": 10696 "bytesInOutput": 10696
@ -52,10 +52,10 @@
"bytesInOutput": 6759 "bytesInOutput": 6759
}, },
"demo/browser.js": { "demo/browser.js": {
"bytesInOutput": 17471 "bytesInOutput": 17538
} }
}, },
"bytes": 1388055 "bytes": 1388062
} }
} }
} }

8
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

528
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

26
dist/human.esm.json vendored
View File

@ -56,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14094, "bytes": 14034,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
] ]
}, },
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytes": 3095, "bytes": 3061,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -111,7 +111,7 @@
] ]
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytes": 2890, "bytes": 2837,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -438,7 +438,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 10422, "bytes": 10308,
"imports": [] "imports": []
}, },
"src/sample.ts": { "src/sample.ts": {
@ -446,7 +446,7 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2581, "bytes": 2559,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1980000 "bytes": 1979712
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -563,7 +563,7 @@
"entryPoint": "src/human.ts", "entryPoint": "src/human.ts",
"inputs": { "inputs": {
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytesInOutput": 1546 "bytesInOutput": 1519
}, },
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1690 "bytesInOutput": 1690
@ -575,7 +575,7 @@
"bytesInOutput": 394 "bytesInOutput": 394
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056721 "bytesInOutput": 1056717
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -593,13 +593,13 @@
"bytesInOutput": 28983 "bytesInOutput": 28983
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5043 "bytesInOutput": 5112
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11955 "bytesInOutput": 11955
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576 "bytesInOutput": 1535
}, },
"src/profile.ts": { "src/profile.ts": {
"bytesInOutput": 606 "bytesInOutput": 606
@ -677,19 +677,19 @@
"bytesInOutput": 2355 "bytesInOutput": 2355
}, },
"config.js": { "config.js": {
"bytesInOutput": 1439 "bytesInOutput": 1422
}, },
"src/sample.ts": { "src/sample.ts": {
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2583 "bytesInOutput": 2541
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9740 "bytesInOutput": 9740
} }
}, },
"bytes": 1353240 "bytes": 1353178
} }
} }
} }

26
dist/human.iife.json vendored
View File

@ -56,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14094, "bytes": 14034,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
] ]
}, },
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytes": 3095, "bytes": 3061,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -111,7 +111,7 @@
] ]
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytes": 2890, "bytes": 2837,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -438,7 +438,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 10422, "bytes": 10308,
"imports": [] "imports": []
}, },
"src/sample.ts": { "src/sample.ts": {
@ -446,7 +446,7 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2581, "bytes": 2559,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1980011 "bytes": 1979723
}, },
"dist/human.ts": { "dist/human.ts": {
"imports": [], "imports": [],
@ -561,7 +561,7 @@
"entryPoint": "src/human.ts", "entryPoint": "src/human.ts",
"inputs": { "inputs": {
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytesInOutput": 1546 "bytesInOutput": 1519
}, },
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1690 "bytesInOutput": 1690
@ -576,7 +576,7 @@
"bytesInOutput": 394 "bytesInOutput": 394
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056721 "bytesInOutput": 1056717
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -594,10 +594,10 @@
"bytesInOutput": 28983 "bytesInOutput": 28983
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5043 "bytesInOutput": 5112
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576 "bytesInOutput": 1535
}, },
"src/profile.ts": { "src/profile.ts": {
"bytesInOutput": 606 "bytesInOutput": 606
@ -675,19 +675,19 @@
"bytesInOutput": 2355 "bytesInOutput": 2355
}, },
"config.js": { "config.js": {
"bytesInOutput": 1439 "bytesInOutput": 1422
}, },
"src/sample.ts": { "src/sample.ts": {
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2583 "bytesInOutput": 2541
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9740 "bytesInOutput": 9740
} }
}, },
"bytes": 1353282 "bytes": 1353220
} }
} }
} }

8
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

24
dist/human.node.json vendored
View File

@ -56,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14094, "bytes": 14034,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
] ]
}, },
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytes": 3095, "bytes": 3061,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -111,7 +111,7 @@
] ]
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytes": 2890, "bytes": 2837,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -438,7 +438,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 10422, "bytes": 10308,
"imports": [] "imports": []
}, },
"src/sample.ts": { "src/sample.ts": {
@ -446,7 +446,7 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2581, "bytes": 2559,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 746751 "bytes": 746468
}, },
"dist/human.node-gpu.js": { "dist/human.node-gpu.js": {
"imports": [], "imports": [],
@ -564,7 +564,7 @@
"bytesInOutput": 598 "bytesInOutput": 598
}, },
"src/blazeface/facemesh.ts": { "src/blazeface/facemesh.ts": {
"bytesInOutput": 1585 "bytesInOutput": 1558
}, },
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1677 "bytesInOutput": 1677
@ -585,7 +585,7 @@
"bytesInOutput": 2338 "bytesInOutput": 2338
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5090 "bytesInOutput": 5159
}, },
"src/blazeface/box.ts": { "src/blazeface/box.ts": {
"bytesInOutput": 854 "bytesInOutput": 854
@ -597,7 +597,7 @@
"bytesInOutput": 28973 "bytesInOutput": 28973
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1618 "bytesInOutput": 1577
}, },
"src/profile.ts": { "src/profile.ts": {
"bytesInOutput": 604 "bytesInOutput": 604
@ -675,19 +675,19 @@
"bytesInOutput": 10973 "bytesInOutput": 10973
}, },
"config.js": { "config.js": {
"bytesInOutput": 1438 "bytesInOutput": 1421
}, },
"src/sample.ts": { "src/sample.ts": {
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2580 "bytesInOutput": 2538
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9629 "bytesInOutput": 9629
} }
}, },
"bytes": 290654 "bytes": 290596
} }
} }
} }

528
dist/human.ts vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.ts.map vendored
View File

File diff suppressed because one or more lines are too long

4435
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -20,7 +20,6 @@
"type": "git", "type": "git",
"url": "git+https://github.com/vladmandic/human.git" "url": "git+https://github.com/vladmandic/human.git"
}, },
"dependencies": {},
"peerDependencies": {}, "peerDependencies": {},
"devDependencies": { "devDependencies": {
"@tensorflow/tfjs": "^3.2.0", "@tensorflow/tfjs": "^3.2.0",

2
src/blazeface/facemesh.ts
View File

@ -24,7 +24,7 @@ export class MediaPipeFaceMesh {
if (mesh && mesh.length > 0) { if (mesh && mesh.length > 0) {
for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]); for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
} }
const boxRaw = (config.face.mesh.returnRawData && prediction.box) ? { topLeft: prediction.box.startPoint, bottomRight: prediction.box.endPoint } : null; const boxRaw = (prediction.box) ? { topLeft: prediction.box.startPoint, bottomRight: prediction.box.endPoint } : null;
const box = prediction.box ? [ const box = prediction.box ? [
Math.max(0, prediction.box.startPoint[0]), Math.max(0, prediction.box.startPoint[0]),
Math.max(0, prediction.box.startPoint[1]), Math.max(0, prediction.box.startPoint[1]),

4
src/blazeface/facepipeline.ts
View File

@ -180,7 +180,6 @@ export class Pipeline {
}); });
} }
// console.log('face', `skipped: ${this.skipped} max: ${config.face.detector.maxFaces} detected: ${this.detectedFaces} stored: ${this.storedBoxes.length} new: ${detector?.boxes?.length}`);
let results = tf.tidy(() => this.storedBoxes.map((box, i) => { let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box). // The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
let face; let face;
@ -255,12 +254,13 @@ export class Pipeline {
image: face, image: face,
rawCoords, rawCoords,
}; };
if (!config.face.mesh.returnRawData) delete prediction.rawCoords;
this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence }; this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence };
return prediction; return prediction;
})); }));
results = results.filter((a) => a !== null); results = results.filter((a) => a !== null);
// remove cache entries for detected boxes on low confidence
if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config.face.detector.minConfidence);
this.detectedFaces = results.length; this.detectedFaces = results.length;
return results; return results;
} }

2
src/faceboxes/faceboxes.ts
View File

@ -51,7 +51,7 @@ export class FaceBoxes {
const resized = tf.image.cropAndResize(input, [crop], [0], [this.config.face.detector.inputSize, this.config.face.detector.inputSize]); const resized = tf.image.cropAndResize(input, [crop], [0], [this.config.face.detector.inputSize, this.config.face.detector.inputSize]);
const image = resized.div([255]); const image = resized.div([255]);
resized.dispose(); resized.dispose();
results.push({ confidence: scores[i], box, boxRaw: this.config.face.mesh.returnRawData ? boxRaw : null, image }); results.push({ confidence: scores[i], box, boxRaw, image });
// add mesh, meshRaw, annotations, // add mesh, meshRaw, annotations,
} }
} }

2
wiki

@ -1 +1 @@
Subproject commit 0b8afcd5149686d64d6f00bebe33937cd175d949 Subproject commit f981107e84085bb5ac93441162e900a0d73d0503