mirror of https://github.com/vladmandic/human
reimplemented blazeface processing
Vladimir Mandic
parent
1cad163327
commit
a47e45b855
|
|
@ -71,10 +71,10 @@ export default {
|
|||
// e.g., if model is running st 25 FPS, we can re-use existing bounding
|
||||
// box for updated face analysis as the head probably hasn't moved much
|
||||
// in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.1, // threshold for discarding a prediction
|
||||
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
iouThreshold: 0.2, // threshold for deciding whether boxes overlap too much in
|
||||
// non-maximum suppression (0.1 means drop if overlap 10%)
|
||||
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
|
||||
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on score
|
||||
// in non-maximum suppression,
|
||||
// this is applied on detection objects only and before minConfidence
|
||||
},
|
||||
|
|
|
|||
|
|
@ -393,8 +393,7 @@ async function main() {
|
|||
// this is not required, just pre-warms all models for faster initial inference
|
||||
if (ui.modelsWarmup) {
|
||||
status('initializing');
|
||||
const warmup = new ImageData(256, 256);
|
||||
await human.detect(warmup, userConfig);
|
||||
await human.warmup(userConfig);
|
||||
}
|
||||
status('human: ready');
|
||||
document.getElementById('loader').style.display = 'none';
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@
|
|||
<!-- <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgpu@0.0.1-alpha.0/dist/tf-webgpu.js"></script> -->
|
||||
<!-- load compiled demo js -->
|
||||
<script src="../dist/demo-browser-index.js"></script>
|
||||
<!-- alternatively load demo sources directly -->
|
||||
<!-- <script src="browser.js" type="module"></script> -->
|
||||
<!-- alternatively load demo sources directly, this is not supported on mobile platforms as they don't support type=module -->
|
||||
<!-- <script src="./browser.js" type="module"></script> -->
|
||||
<style>
|
||||
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 400; src: local('Lato'), url('../assets/lato.ttf') format('truetype'); }
|
||||
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"demo/browser.js": {
|
||||
"bytes": 18563,
|
||||
"bytes": 18511,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/human.esm.js"
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"bytes": 1278633,
|
||||
"bytes": 1278581,
|
||||
"imports": []
|
||||
}
|
||||
},
|
||||
|
|
@ -31,13 +31,13 @@
|
|||
"dist/demo-browser-index.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 5535482
|
||||
"bytes": 5534501
|
||||
},
|
||||
"dist/demo-browser-index.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 1665139
|
||||
"bytesInOutput": 1665015
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 8716
|
||||
|
|
@ -49,10 +49,10 @@
|
|||
"bytesInOutput": 13425
|
||||
},
|
||||
"demo/browser.js": {
|
||||
"bytesInOutput": 16209
|
||||
"bytesInOutput": 16157
|
||||
}
|
||||
},
|
||||
"bytes": 1712507
|
||||
"bytes": 1712331
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -128,11 +128,11 @@
|
|||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytes": 7058,
|
||||
"bytes": 6937,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"bytes": 1924,
|
||||
"bytes": 1929,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 14752,
|
||||
"bytes": 13695,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/face/box.js"
|
||||
|
|
@ -194,7 +194,7 @@
|
|||
]
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytes": 3233,
|
||||
"bytes": 3306,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
|
|
@ -214,7 +214,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7445,
|
||||
"bytes": 7541,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
|
|
@ -243,7 +243,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 13770,
|
||||
"bytes": 13918,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/face/facemesh.js"
|
||||
|
|
@ -281,7 +281,7 @@
|
|||
]
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytes": 4604,
|
||||
"bytes": 4648,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/imagefx.js"
|
||||
|
|
@ -301,13 +301,13 @@
|
|||
"dist/human.esm-nobundle.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 621753
|
||||
"bytes": 620819
|
||||
},
|
||||
"dist/human.esm-nobundle.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 3093
|
||||
"bytesInOutput": 3103
|
||||
},
|
||||
"src/face/keypoints.js": {
|
||||
"bytesInOutput": 1945
|
||||
|
|
@ -319,7 +319,7 @@
|
|||
"bytesInOutput": 1171
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 5647
|
||||
"bytesInOutput": 5432
|
||||
},
|
||||
"src/face/uvcoords.js": {
|
||||
"bytesInOutput": 16785
|
||||
|
|
@ -385,7 +385,7 @@
|
|||
"bytesInOutput": 997
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 2741
|
||||
"bytesInOutput": 2781
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127000
|
||||
|
|
@ -394,13 +394,13 @@
|
|||
"bytesInOutput": 1219
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytesInOutput": 1574
|
||||
"bytesInOutput": 1601
|
||||
},
|
||||
"src/imagefx.js": {
|
||||
"bytesInOutput": 11013
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytesInOutput": 2349
|
||||
"bytesInOutput": 2343
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1279
|
||||
|
|
@ -409,13 +409,13 @@
|
|||
"bytesInOutput": 3047
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7256
|
||||
"bytesInOutput": 7348
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 0
|
||||
}
|
||||
},
|
||||
"bytes": 216959
|
||||
"bytes": 216907
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -288,7 +288,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytes": 7058,
|
||||
"bytes": 6937,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -296,7 +296,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"bytes": 1924,
|
||||
"bytes": 1929,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -327,7 +327,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 14752,
|
||||
"bytes": 13695,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -371,7 +371,7 @@
|
|||
]
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytes": 3233,
|
||||
"bytes": 3306,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
|
|
@ -398,7 +398,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7445,
|
||||
"bytes": 7541,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -433,7 +433,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 13770,
|
||||
"bytes": 13918,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -474,7 +474,7 @@
|
|||
]
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytes": 4604,
|
||||
"bytes": 4648,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -513,7 +513,7 @@
|
|||
"dist/human.esm.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 5417738
|
||||
"bytes": 5416804
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"imports": [],
|
||||
|
|
@ -576,7 +576,7 @@
|
|||
"bytesInOutput": 760
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 3103
|
||||
"bytesInOutput": 3113
|
||||
},
|
||||
"src/face/keypoints.js": {
|
||||
"bytesInOutput": 1946
|
||||
|
|
@ -588,7 +588,7 @@
|
|||
"bytesInOutput": 1190
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 5639
|
||||
"bytesInOutput": 5425
|
||||
},
|
||||
"src/face/uvcoords.js": {
|
||||
"bytesInOutput": 16786
|
||||
|
|
@ -654,7 +654,7 @@
|
|||
"bytesInOutput": 1005
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 2742
|
||||
"bytesInOutput": 2782
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127001
|
||||
|
|
@ -663,13 +663,13 @@
|
|||
"bytesInOutput": 1197
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytesInOutput": 1575
|
||||
"bytesInOutput": 1602
|
||||
},
|
||||
"src/imagefx.js": {
|
||||
"bytesInOutput": 11014
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytesInOutput": 2365
|
||||
"bytesInOutput": 2358
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1280
|
||||
|
|
@ -678,13 +678,13 @@
|
|||
"bytesInOutput": 3048
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7274
|
||||
"bytesInOutput": 7366
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 0
|
||||
}
|
||||
},
|
||||
"bytes": 1278633
|
||||
"bytes": 1278581
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -288,7 +288,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytes": 7058,
|
||||
"bytes": 6937,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -296,7 +296,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"bytes": 1924,
|
||||
"bytes": 1929,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -327,7 +327,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 14752,
|
||||
"bytes": 13695,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -371,7 +371,7 @@
|
|||
]
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytes": 3233,
|
||||
"bytes": 3306,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
|
|
@ -398,7 +398,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7445,
|
||||
"bytes": 7541,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -433,7 +433,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 13770,
|
||||
"bytes": 13918,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -474,7 +474,7 @@
|
|||
]
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytes": 4604,
|
||||
"bytes": 4648,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -513,7 +513,7 @@
|
|||
"dist/human.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 5417734
|
||||
"bytes": 5416800
|
||||
},
|
||||
"dist/human.js": {
|
||||
"imports": [],
|
||||
|
|
@ -576,7 +576,7 @@
|
|||
"bytesInOutput": 760
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 3103
|
||||
"bytesInOutput": 3113
|
||||
},
|
||||
"src/face/keypoints.js": {
|
||||
"bytesInOutput": 1946
|
||||
|
|
@ -588,7 +588,7 @@
|
|||
"bytesInOutput": 1190
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 5639
|
||||
"bytesInOutput": 5425
|
||||
},
|
||||
"src/face/uvcoords.js": {
|
||||
"bytesInOutput": 16786
|
||||
|
|
@ -654,7 +654,7 @@
|
|||
"bytesInOutput": 1005
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 2742
|
||||
"bytesInOutput": 2782
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127001
|
||||
|
|
@ -663,13 +663,13 @@
|
|||
"bytesInOutput": 1197
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytesInOutput": 1575
|
||||
"bytesInOutput": 1602
|
||||
},
|
||||
"src/imagefx.js": {
|
||||
"bytesInOutput": 11014
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytesInOutput": 2365
|
||||
"bytesInOutput": 2358
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1280
|
||||
|
|
@ -678,10 +678,10 @@
|
|||
"bytesInOutput": 3047
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7312
|
||||
"bytesInOutput": 7404
|
||||
}
|
||||
},
|
||||
"bytes": 1278678
|
||||
"bytes": 1278626
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -128,11 +128,11 @@
|
|||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytes": 7058,
|
||||
"bytes": 6937,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"bytes": 1924,
|
||||
"bytes": 1929,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
|
|
@ -156,7 +156,7 @@
|
|||
]
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 14752,
|
||||
"bytes": 13695,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/face/box.js"
|
||||
|
|
@ -194,7 +194,7 @@
|
|||
]
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytes": 3233,
|
||||
"bytes": 3306,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
|
|
@ -214,7 +214,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7445,
|
||||
"bytes": 7541,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
|
|
@ -243,7 +243,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 13770,
|
||||
"bytes": 13918,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/face/facemesh.js"
|
||||
|
|
@ -281,7 +281,7 @@
|
|||
]
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytes": 4604,
|
||||
"bytes": 4648,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/imagefx.js"
|
||||
|
|
@ -301,13 +301,13 @@
|
|||
"dist/human.node-nobundle.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 635930
|
||||
"bytes": 635150
|
||||
},
|
||||
"dist/human.node-nobundle.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 3093
|
||||
"bytesInOutput": 3103
|
||||
},
|
||||
"src/face/keypoints.js": {
|
||||
"bytesInOutput": 1945
|
||||
|
|
@ -319,7 +319,7 @@
|
|||
"bytesInOutput": 1171
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 5647
|
||||
"bytesInOutput": 5432
|
||||
},
|
||||
"src/face/uvcoords.js": {
|
||||
"bytesInOutput": 16785
|
||||
|
|
@ -385,7 +385,7 @@
|
|||
"bytesInOutput": 996
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 2741
|
||||
"bytesInOutput": 2781
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127000
|
||||
|
|
@ -394,13 +394,13 @@
|
|||
"bytesInOutput": 1219
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytesInOutput": 1574
|
||||
"bytesInOutput": 1601
|
||||
},
|
||||
"src/imagefx.js": {
|
||||
"bytesInOutput": 11013
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytesInOutput": 2349
|
||||
"bytesInOutput": 2343
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1278
|
||||
|
|
@ -412,10 +412,10 @@
|
|||
"bytesInOutput": 28
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7256
|
||||
"bytesInOutput": 7348
|
||||
}
|
||||
},
|
||||
"bytes": 216966
|
||||
"bytes": 216914
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,21 +110,17 @@ class BlazeFaceModel {
|
|||
return vals;
|
||||
});
|
||||
|
||||
const scoresVal = scores.dataSync();
|
||||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boundingBox = boundingBoxes[i];
|
||||
const box = createBox(boundingBox);
|
||||
for (const i in boundingBoxes) {
|
||||
const boxIndex = boxIndices[i];
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const sliced = tf.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]);
|
||||
const squeezed = sliced.squeeze();
|
||||
const landmarks = squeezed.reshape([NUM_LANDMARKS, -1]);
|
||||
const probability = tf.slice(scores, [boxIndex], [1]);
|
||||
const annotatedBox = { box, landmarks, probability, anchor };
|
||||
annotatedBoxes.push(annotatedBox);
|
||||
sliced.dispose();
|
||||
squeezed.dispose();
|
||||
// landmarks.dispose();
|
||||
const confidence = scoresVal[boxIndex];
|
||||
if (confidence > this.config.detector.minConfidence) {
|
||||
const box = createBox(boundingBoxes[i]);
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf.tidy(() => tf.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]).squeeze().reshape([NUM_LANDMARKS, -1]));
|
||||
annotatedBoxes.push({ box, landmarks, anchor, confidence });
|
||||
}
|
||||
}
|
||||
detectedOutputs.dispose();
|
||||
boxes.dispose();
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ function scaleBoxCoordinates(box, factor) {
|
|||
return { startPoint, endPoint };
|
||||
}
|
||||
exports.scaleBoxCoordinates = scaleBoxCoordinates;
|
||||
|
||||
function getBoxSize(box) {
|
||||
return [
|
||||
Math.abs(box.endPoint[0] - box.startPoint[0]),
|
||||
|
|
@ -13,6 +14,7 @@ function getBoxSize(box) {
|
|||
];
|
||||
}
|
||||
exports.getBoxSize = getBoxSize;
|
||||
|
||||
function getBoxCenter(box) {
|
||||
return [
|
||||
box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2,
|
||||
|
|
@ -20,6 +22,7 @@ function getBoxCenter(box) {
|
|||
];
|
||||
}
|
||||
exports.getBoxCenter = getBoxCenter;
|
||||
|
||||
function cutBoxFromImageAndResize(box, image, cropSize) {
|
||||
const h = image.shape[1];
|
||||
const w = image.shape[2];
|
||||
|
|
@ -30,6 +33,7 @@ function cutBoxFromImageAndResize(box, image, cropSize) {
|
|||
return tf.image.cropAndResize(image, boxes, [0], cropSize);
|
||||
}
|
||||
exports.cutBoxFromImageAndResize = cutBoxFromImageAndResize;
|
||||
|
||||
function enlargeBox(box, factor = 1.5) {
|
||||
const center = getBoxCenter(box);
|
||||
const size = getBoxSize(box);
|
||||
|
|
@ -39,6 +43,7 @@ function enlargeBox(box, factor = 1.5) {
|
|||
return { startPoint, endPoint, landmarks: box.landmarks };
|
||||
}
|
||||
exports.enlargeBox = enlargeBox;
|
||||
|
||||
function squarifyBox(box) {
|
||||
const centers = getBoxCenter(box);
|
||||
const size = getBoxSize(box);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ const keypoints = require('./keypoints');
|
|||
const util = require('./util');
|
||||
|
||||
const LANDMARKS_COUNT = 468;
|
||||
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.25;
|
||||
const MESH_MOUTH_INDEX = 13;
|
||||
const MESH_KEYPOINTS_LINE_OF_SYMMETRY_INDICES = [MESH_MOUTH_INDEX, keypoints.MESH_ANNOTATIONS['midwayBetweenEyes'][0]];
|
||||
const BLAZEFACE_MOUTH_INDEX = 3;
|
||||
|
|
@ -41,7 +40,7 @@ function replaceRawCoordinates(rawCoords, newCoords, prefix, keys) {
|
|||
class Pipeline {
|
||||
constructor(boundingBoxDetector, meshDetector, irisModel, config) {
|
||||
// An array of facial bounding boxes.
|
||||
this.regionsOfInterest = [];
|
||||
this.storedBoxes = [];
|
||||
this.runsWithoutFaceDetector = 0;
|
||||
this.boundingBoxDetector = boundingBoxDetector;
|
||||
this.meshDetector = meshDetector;
|
||||
|
|
@ -50,6 +49,8 @@ class Pipeline {
|
|||
this.meshHeight = config.mesh.inputSize;
|
||||
this.irisSize = config.iris.inputSize;
|
||||
this.irisEnlarge = 2.3;
|
||||
this.skipped = 1000;
|
||||
this.detectedFaces = 0;
|
||||
}
|
||||
|
||||
transformRawCoords(rawCoords, box, angle, rotationMatrix) {
|
||||
|
|
@ -129,35 +130,39 @@ class Pipeline {
|
|||
}
|
||||
|
||||
async predict(input, config) {
|
||||
this.runsWithoutFaceDetector += 1;
|
||||
let useFreshBox = (this.detectedFaces === 0) || (this.detectedFaces !== this.regionsOfInterest.length);
|
||||
this.skipped++;
|
||||
let useFreshBox = false;
|
||||
// run new detector every skipFrames unless we only want box to start with
|
||||
let detector;
|
||||
// but every skipFrames check if detect boxes number changed
|
||||
if (useFreshBox || (this.runsWithoutFaceDetector > config.detector.skipFrames)) detector = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
// if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1
|
||||
if (config.detector.maxFaces > 1 && detector && detector.boxes && detector.boxes.length > 0 && detector.boxes.length !== this.detectedFaces) useFreshBox = true;
|
||||
if ((this.skipped > config.detector.skipFrames) || !config.mesh.enabled) {
|
||||
detector = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
// don't reset on test image
|
||||
if ((input.shape[1] !== 255) && (input.shape[2] !== 255)) this.skipped = 0;
|
||||
}
|
||||
|
||||
// if detector result count doesn't match current working set, use it to reset current working set
|
||||
if (detector && detector.boxes && (detector.boxes.length > 0) && (!config.mesh.enabled || (detector.boxes.length !== this.detectedFaces) && (this.detectedFaces !== config.detector.maxFaces))) {
|
||||
this.storedBoxes = [];
|
||||
this.detectedFaces = 0;
|
||||
for (const possible of detector.boxes) {
|
||||
this.storedBoxes.push({ startPoint: possible.box.startPoint.dataSync(), endPoint: possible.box.endPoint.dataSync(), landmarks: possible.landmarks, confidence: possible.confidence });
|
||||
}
|
||||
if (this.storedBoxes.length > 0) useFreshBox = true;
|
||||
}
|
||||
|
||||
if (useFreshBox) {
|
||||
// const detector = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
if (!detector || !detector.boxes || (detector.boxes.length === 0)) {
|
||||
this.regionsOfInterest = [];
|
||||
this.storedBoxes = [];
|
||||
this.detectedFaces = 0;
|
||||
return null;
|
||||
}
|
||||
const scaledBoxes = detector.boxes.map((prediction) => {
|
||||
const startPoint = prediction.box.startPoint.squeeze();
|
||||
const endPoint = prediction.box.endPoint.squeeze();
|
||||
const predictionBox = {
|
||||
startPoint: startPoint.arraySync(),
|
||||
endPoint: endPoint.arraySync(),
|
||||
};
|
||||
startPoint.dispose();
|
||||
endPoint.dispose();
|
||||
const scaledBox = bounding.scaleBoxCoordinates(predictionBox, detector.scaleFactor);
|
||||
for (const i in this.storedBoxes) {
|
||||
const scaledBox = bounding.scaleBoxCoordinates({ startPoint: this.storedBoxes[i].startPoint, endPoint: this.storedBoxes[i].endPoint }, detector.scaleFactor);
|
||||
const enlargedBox = bounding.enlargeBox(scaledBox);
|
||||
const landmarks = prediction.landmarks.arraySync();
|
||||
return { ...enlargedBox, landmarks };
|
||||
});
|
||||
this.updateRegionsOfInterest(scaledBoxes);
|
||||
const landmarks = this.storedBoxes[i].landmarks.arraySync();
|
||||
const confidence = this.storedBoxes[i].confidence;
|
||||
this.storedBoxes[i] = { ...enlargedBox, confidence, landmarks };
|
||||
}
|
||||
this.runsWithoutFaceDetector = 0;
|
||||
}
|
||||
if (detector && detector.boxes) {
|
||||
|
|
@ -165,10 +170,12 @@ class Pipeline {
|
|||
prediction.box.startPoint.dispose();
|
||||
prediction.box.endPoint.dispose();
|
||||
prediction.landmarks.dispose();
|
||||
prediction.probability.dispose();
|
||||
});
|
||||
}
|
||||
let results = tf.tidy(() => this.regionsOfInterest.map((box, i) => {
|
||||
|
||||
// console.log(this.skipped, config.detector.skipFrames, this.detectedFaces, config.detector.maxFaces, detector?.boxes.length, this.storedBoxes.length);
|
||||
|
||||
let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
|
||||
let angle = 0;
|
||||
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
|
||||
const boxLandmarksFromMeshModel = box.landmarks.length >= LANDMARKS_COUNT;
|
||||
|
|
@ -187,6 +194,19 @@ class Pipeline {
|
|||
}
|
||||
const boxCPU = { startPoint: box.startPoint, endPoint: box.endPoint };
|
||||
const face = bounding.cutBoxFromImageAndResize(boxCPU, rotatedImage, [this.meshHeight, this.meshWidth]).div(255);
|
||||
|
||||
// if we're not going to produce mesh, don't spend time with further processing
|
||||
if (!config.mesh.enabled) {
|
||||
const prediction = {
|
||||
coords: null,
|
||||
box,
|
||||
faceConfidence: null,
|
||||
confidence: box.confidence,
|
||||
image: face,
|
||||
};
|
||||
return prediction;
|
||||
}
|
||||
|
||||
// The first returned tensor represents facial contours, which are included in the coordinates.
|
||||
const [, confidence, coords] = this.meshDetector.predict(face);
|
||||
const confidenceVal = confidence.dataSync()[0];
|
||||
|
|
@ -224,17 +244,15 @@ class Pipeline {
|
|||
const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
|
||||
tf.dispose(rawCoords);
|
||||
const landmarksBox = bounding.enlargeBox(this.calculateLandmarksBoundingBox(transformedCoordsData));
|
||||
const transformedCoords = tf.tensor2d(transformedCoordsData);
|
||||
const prediction = {
|
||||
coords: null,
|
||||
coords: transformedCoords,
|
||||
box: landmarksBox,
|
||||
confidence: confidenceVal,
|
||||
faceConfidence: confidenceVal,
|
||||
confidence: box.confidence,
|
||||
image: face,
|
||||
};
|
||||
if (config.mesh.enabled) {
|
||||
const transformedCoords = tf.tensor2d(transformedCoordsData);
|
||||
this.regionsOfInterest[i] = { ...landmarksBox, landmarks: transformedCoords.arraySync() };
|
||||
prediction.coords = transformedCoords;
|
||||
}
|
||||
this.storedBoxes[i] = { ...landmarksBox, landmarks: transformedCoords.arraySync(), confidence: box.confidence, faceConfidence: confidenceVal };
|
||||
return prediction;
|
||||
}));
|
||||
results = results.filter((a) => a !== null);
|
||||
|
|
@ -242,42 +260,6 @@ class Pipeline {
|
|||
return results;
|
||||
}
|
||||
|
||||
// Updates regions of interest if the intersection over union between the incoming and previous regions falls below a threshold.
|
||||
updateRegionsOfInterest(boxes) {
|
||||
for (let i = 0; i < boxes.length; i++) {
|
||||
const box = boxes[i];
|
||||
const previousBox = this.regionsOfInterest[i];
|
||||
let iou = 0;
|
||||
if (previousBox && previousBox.startPoint) {
|
||||
const [boxStartX, boxStartY] = box.startPoint;
|
||||
const [boxEndX, boxEndY] = box.endPoint;
|
||||
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;
|
||||
const [previousBoxEndX, previousBoxEndY] = previousBox.endPoint;
|
||||
const xStartMax = Math.max(boxStartX, previousBoxStartX);
|
||||
const yStartMax = Math.max(boxStartY, previousBoxStartY);
|
||||
const xEndMin = Math.min(boxEndX, previousBoxEndX);
|
||||
const yEndMin = Math.min(boxEndY, previousBoxEndY);
|
||||
const intersection = (xEndMin - xStartMax) * (yEndMin - yStartMax);
|
||||
const boxArea = (boxEndX - boxStartX) * (boxEndY - boxStartY);
|
||||
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
|
||||
iou = intersection / (boxArea + previousBoxArea - intersection);
|
||||
}
|
||||
if (iou < UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD) {
|
||||
this.regionsOfInterest[i] = box;
|
||||
}
|
||||
}
|
||||
this.regionsOfInterest = this.regionsOfInterest.slice(0, boxes.length);
|
||||
}
|
||||
|
||||
clearRegionOfInterest(index) {
|
||||
if (this.regionsOfInterest[index] != null) {
|
||||
this.regionsOfInterest = [
|
||||
...this.regionsOfInterest.slice(0, index),
|
||||
...this.regionsOfInterest.slice(index + 1),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
calculateLandmarksBoundingBox(landmarks) {
|
||||
const xs = landmarks.map((d) => d[0]);
|
||||
const ys = landmarks.map((d) => d[1]);
|
||||
|
|
|
|||
|
|
@ -25,17 +25,19 @@ exports.face = (res) => {
|
|||
// if (face.annotations['rightCheek'] && face.annotations['leftCheek'] && (face.annotations['rightCheek'].length > 0) && (face.annotations['leftCheek'].length > 0)) {
|
||||
// gestures.push(`facing ${((face.annotations['rightCheek'][0][2] > 0) || (face.annotations['leftCheek'][0][2] < 0)) ? 'right' : 'left'}`);
|
||||
// }
|
||||
const eyeFacing = face.mesh[35][2] - face.mesh[263][2];
|
||||
if (Math.abs(eyeFacing) < 10) gestures.push('facing camera');
|
||||
else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`);
|
||||
const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
|
||||
if (openLeft < 0.2) gestures.push('blink left eye');
|
||||
const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
|
||||
if (openRight < 0.2) gestures.push('blink right eye');
|
||||
const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1]));
|
||||
if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`);
|
||||
const chinDepth = face.mesh[152][2];
|
||||
if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`);
|
||||
if (face.mesh && face.mesh.length > 0) {
|
||||
const eyeFacing = face.mesh[35][2] - face.mesh[263][2];
|
||||
if (Math.abs(eyeFacing) < 10) gestures.push('facing camera');
|
||||
else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`);
|
||||
const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
|
||||
if (openLeft < 0.2) gestures.push('blink left eye');
|
||||
const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
|
||||
if (openRight < 0.2) gestures.push('blink right eye');
|
||||
const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1]));
|
||||
if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`);
|
||||
const chinDepth = face.mesh[152][2];
|
||||
if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`);
|
||||
}
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class HandPipeline {
|
|||
this.meshDetector = meshDetector;
|
||||
this.inputSize = inputSize;
|
||||
this.storedBoxes = [];
|
||||
this.skipped = 0;
|
||||
this.skipped = 1000;
|
||||
this.detectedHands = 0;
|
||||
}
|
||||
|
||||
|
|
@ -91,7 +91,8 @@ class HandPipeline {
|
|||
let boxes;
|
||||
if ((this.skipped > config.skipFrames) || !config.landmarks) {
|
||||
boxes = await this.boxDetector.estimateHandBounds(image, config);
|
||||
this.skipped = 0;
|
||||
// don't reset on test image
|
||||
if ((image.shape[1] !== 255) && (image.shape[2] !== 255)) this.skipped = 0;
|
||||
}
|
||||
|
||||
// if detector result count doesn't match current working set, use it to reset current working set
|
||||
|
|
|
|||
|
|
@ -373,6 +373,12 @@ class Human {
|
|||
resolve({ face: faceRes, body: poseRes, hand: handRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas });
|
||||
});
|
||||
}
|
||||
|
||||
async warmup(userConfig) {
|
||||
const warmup = new ImageData(255, 255);
|
||||
await this.detect(warmup, userConfig);
|
||||
this.log('warmed up');
|
||||
}
|
||||
}
|
||||
|
||||
export { Human as default };
|
||||
|
|
|
|||
|
|
@ -52,8 +52,10 @@ function process(input, config) {
|
|||
if (config.filter.polaroid) this.fx.addFilter('polaroid');
|
||||
if (config.filter.pixelate !== 0) this.fx.addFilter('pixelate', config.filter.pixelate);
|
||||
this.fx.apply(inCanvas);
|
||||
} else {
|
||||
outCanvas = inCanvas;
|
||||
}
|
||||
if (!outCanvas) outCanvas = inCanvas;
|
||||
// if (!outCanvas) outCanvas = inCanvas;
|
||||
let pixels;
|
||||
if ((config.backend === 'webgl') || (outCanvas instanceof ImageData)) {
|
||||
// tf kernel-optimized method to get imagedata, also if input is imagedata, just use it
|
||||
|
|
|
|||
Loading…
Reference in New Issue