ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix performance issue when running with low confidence

pull/91/head
Vladimir Mandic 2021-03-08 10:06:34 -05:00
parent 578195f472
commit 3588007f1c
24 changed files with 4958 additions and 1096 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.js
View File

@ -75,7 +75,7 @@ export default {
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
maxFaces: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance
skipFrames: 11, // how many frames to go without re-running the face bounding box detector
skipFrames: 21, // how many frames to go without re-running the face bounding box detector
// only used for video inputs
// e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated face analysis as the head probably hasn't moved much
@ -94,7 +94,6 @@ export default {
enabled: true,
modelPath: '../models/facemesh.json',
inputSize: 192, // fixed value
returnRawData: false, // in addition to standard mesh and box values, return raw normalized values as well
},
iris: {
@ -116,7 +115,7 @@ export default {
minConfidence: 0.1, // threshold for discarding a prediction
modelPath: '../models/gender.json', // can be 'gender' or 'gender-ssrnet-imdb'
inputSize: 64, // fixed value
skipFrames: 41, // how many frames to go without re-running the detector
skipFrames: 32, // how many frames to go without re-running the detector
// only used for video inputs
},
@ -124,7 +123,7 @@ export default {
enabled: true,
inputSize: 64, // fixed value
minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 21, // how many frames to go without re-running the detector
skipFrames: 33, // how many frames to go without re-running the detector
modelPath: '../models/emotion.json',
},

7
demo/browser.js
View File

@ -6,13 +6,13 @@ const userConfig = { backend: 'webgl' }; // add any user configuration overrides
/*
const userConfig = {
backend: 'wasm',
backend: 'webgl',
async: false,
warmup: 'face',
videoOptimized: false,
videoOptimized: true,
face: { enabled: true, iris: { enabled: false }, mesh: { enabled: true }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
hand: { enabled: false },
gestures: { enabled: true },
gesture: { enabled: false },
body: { enabled: false, modelType: 'blazepose', modelPath: '../models/blazepose.json' },
};
*/
@ -424,6 +424,7 @@ function setupMenu() {
});
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('draw with curves', human.draw.options, 'useCurves');
menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');

402
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

12
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"dist/human.esm.js": {
"bytes": 1353240,
"bytes": 1353178,
"imports": []
},
"demo/menu.js": {
@ -13,7 +13,7 @@
"imports": []
},
"demo/browser.js": {
"bytes": 27966,
"bytes": 28043,
"imports": [
{
"path": "dist/human.esm.js",
@ -35,7 +35,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 2061718
"bytes": 2061556
},
"dist/demo-browser-index.js": {
"imports": [],
@ -43,7 +43,7 @@
"entryPoint": "demo/browser.js",
"inputs": {
"dist/human.esm.js": {
"bytesInOutput": 1345744
"bytesInOutput": 1345684
},
"demo/menu.js": {
"bytesInOutput": 10696
@ -52,10 +52,10 @@
"bytesInOutput": 6759
},
"demo/browser.js": {
"bytesInOutput": 17471
"bytesInOutput": 17538
}
},
"bytes": 1388055
"bytes": 1388062
}
}
}

8
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

528
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

26
dist/human.esm.json vendored
View File

@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14094,
"bytes": 14034,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
]
},
"src/blazeface/facemesh.ts": {
"bytes": 3095,
"bytes": 3061,
"imports": [
{
"path": "src/log.ts",
@ -111,7 +111,7 @@
]
},
"src/faceboxes/faceboxes.ts": {
"bytes": 2890,
"bytes": 2837,
"imports": [
{
"path": "src/log.ts",
@ -438,7 +438,7 @@
]
},
"config.js": {
"bytes": 10422,
"bytes": 10308,
"imports": []
},
"src/sample.ts": {
@ -446,7 +446,7 @@
"imports": []
},
"package.json": {
"bytes": 2581,
"bytes": 2559,
"imports": []
},
"src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 1980000
"bytes": 1979712
},
"dist/human.esm.js": {
"imports": [],
@ -563,7 +563,7 @@
"entryPoint": "src/human.ts",
"inputs": {
"src/blazeface/facemesh.ts": {
"bytesInOutput": 1546
"bytesInOutput": 1519
},
"src/posenet/keypoints.ts": {
"bytesInOutput": 1690
@ -575,7 +575,7 @@
"bytesInOutput": 394
},
"dist/tfjs.esm.js": {
"bytesInOutput": 1056721
"bytesInOutput": 1056717
},
"src/tfjs/backend.ts": {
"bytesInOutput": 1053
@ -593,13 +593,13 @@
"bytesInOutput": 28983
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5043
"bytesInOutput": 5112
},
"src/human.ts": {
"bytesInOutput": 11955
},
"src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576
"bytesInOutput": 1535
},
"src/profile.ts": {
"bytesInOutput": 606
@ -677,19 +677,19 @@
"bytesInOutput": 2355
},
"config.js": {
"bytesInOutput": 1439
"bytesInOutput": 1422
},
"src/sample.ts": {
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2583
"bytesInOutput": 2541
},
"src/draw.ts": {
"bytesInOutput": 9740
}
},
"bytes": 1353240
"bytes": 1353178
}
}
}

26
dist/human.iife.json vendored
View File

@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14094,
"bytes": 14034,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
]
},
"src/blazeface/facemesh.ts": {
"bytes": 3095,
"bytes": 3061,
"imports": [
{
"path": "src/log.ts",
@ -111,7 +111,7 @@
]
},
"src/faceboxes/faceboxes.ts": {
"bytes": 2890,
"bytes": 2837,
"imports": [
{
"path": "src/log.ts",
@ -438,7 +438,7 @@
]
},
"config.js": {
"bytes": 10422,
"bytes": 10308,
"imports": []
},
"src/sample.ts": {
@ -446,7 +446,7 @@
"imports": []
},
"package.json": {
"bytes": 2581,
"bytes": 2559,
"imports": []
},
"src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 1980011
"bytes": 1979723
},
"dist/human.ts": {
"imports": [],
@ -561,7 +561,7 @@
"entryPoint": "src/human.ts",
"inputs": {
"src/blazeface/facemesh.ts": {
"bytesInOutput": 1546
"bytesInOutput": 1519
},
"src/posenet/keypoints.ts": {
"bytesInOutput": 1690
@ -576,7 +576,7 @@
"bytesInOutput": 394
},
"dist/tfjs.esm.js": {
"bytesInOutput": 1056721
"bytesInOutput": 1056717
},
"src/tfjs/backend.ts": {
"bytesInOutput": 1053
@ -594,10 +594,10 @@
"bytesInOutput": 28983
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5043
"bytesInOutput": 5112
},
"src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576
"bytesInOutput": 1535
},
"src/profile.ts": {
"bytesInOutput": 606
@ -675,19 +675,19 @@
"bytesInOutput": 2355
},
"config.js": {
"bytesInOutput": 1439
"bytesInOutput": 1422
},
"src/sample.ts": {
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2583
"bytesInOutput": 2541
},
"src/draw.ts": {
"bytesInOutput": 9740
}
},
"bytes": 1353282
"bytes": 1353220
}
}
}

8
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

24
dist/human.node.json vendored
View File

@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14094,
"bytes": 14034,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -77,7 +77,7 @@
]
},
"src/blazeface/facemesh.ts": {
"bytes": 3095,
"bytes": 3061,
"imports": [
{
"path": "src/log.ts",
@ -111,7 +111,7 @@
]
},
"src/faceboxes/faceboxes.ts": {
"bytes": 2890,
"bytes": 2837,
"imports": [
{
"path": "src/log.ts",
@ -438,7 +438,7 @@
]
},
"config.js": {
"bytes": 10422,
"bytes": 10308,
"imports": []
},
"src/sample.ts": {
@ -446,7 +446,7 @@
"imports": []
},
"package.json": {
"bytes": 2581,
"bytes": 2559,
"imports": []
},
"src/draw.ts": {
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 746751
"bytes": 746468
},
"dist/human.node-gpu.js": {
"imports": [],
@ -564,7 +564,7 @@
"bytesInOutput": 598
},
"src/blazeface/facemesh.ts": {
"bytesInOutput": 1585
"bytesInOutput": 1558
},
"src/posenet/keypoints.ts": {
"bytesInOutput": 1677
@ -585,7 +585,7 @@
"bytesInOutput": 2338
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5090
"bytesInOutput": 5159
},
"src/blazeface/box.ts": {
"bytesInOutput": 854
@ -597,7 +597,7 @@
"bytesInOutput": 28973
},
"src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1618
"bytesInOutput": 1577
},
"src/profile.ts": {
"bytesInOutput": 604
@ -675,19 +675,19 @@
"bytesInOutput": 10973
},
"config.js": {
"bytesInOutput": 1438
"bytesInOutput": 1421
},
"src/sample.ts": {
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2580
"bytesInOutput": 2538
},
"src/draw.ts": {
"bytesInOutput": 9629
}
},
"bytes": 290654
"bytes": 290596
}
}
}

528
dist/human.ts vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.ts.map vendored
View File

File diff suppressed because one or more lines are too long

4435
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -20,7 +20,6 @@
"type": "git",
"url": "git+https://github.com/vladmandic/human.git"
},
"dependencies": {},
"peerDependencies": {},
"devDependencies": {
"@tensorflow/tfjs": "^3.2.0",

2
src/blazeface/facemesh.ts
View File

@ -24,7 +24,7 @@ export class MediaPipeFaceMesh {
if (mesh && mesh.length > 0) {
for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
}
const boxRaw = (config.face.mesh.returnRawData && prediction.box) ? { topLeft: prediction.box.startPoint, bottomRight: prediction.box.endPoint } : null;
const boxRaw = (prediction.box) ? { topLeft: prediction.box.startPoint, bottomRight: prediction.box.endPoint } : null;
const box = prediction.box ? [
Math.max(0, prediction.box.startPoint[0]),
Math.max(0, prediction.box.startPoint[1]),

4
src/blazeface/facepipeline.ts
View File

@ -180,7 +180,6 @@ export class Pipeline {
});
}
// console.log('face', `skipped: ${this.skipped} max: ${config.face.detector.maxFaces} detected: ${this.detectedFaces} stored: ${this.storedBoxes.length} new: ${detector?.boxes?.length}`);
let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
let face;
@ -255,12 +254,13 @@ export class Pipeline {
image: face,
rawCoords,
};
if (!config.face.mesh.returnRawData) delete prediction.rawCoords;
this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence };
return prediction;
}));
results = results.filter((a) => a !== null);
// remove cache entries for detected boxes on low confidence
if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config.face.detector.minConfidence);
this.detectedFaces = results.length;
return results;
}

2
src/faceboxes/faceboxes.ts
View File

@ -51,7 +51,7 @@ export class FaceBoxes {
const resized = tf.image.cropAndResize(input, [crop], [0], [this.config.face.detector.inputSize, this.config.face.detector.inputSize]);
const image = resized.div([255]);
resized.dispose();
results.push({ confidence: scores[i], box, boxRaw: this.config.face.mesh.returnRawData ? boxRaw : null, image });
results.push({ confidence: scores[i], box, boxRaw, image });
// add mesh, meshRaw, annotations,
}
}

2
wiki

@ -1 +1 @@
Subproject commit 0b8afcd5149686d64d6f00bebe33937cd175d949
Subproject commit f981107e84085bb5ac93441162e900a0d73d0503