ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix hand detection performance

pull/50/head
Vladimir Mandic 2020-11-07 11:25:03 -05:00
parent 94986ced0d
commit bfc180ebbf
13 changed files with 64 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
assets/lato.ttf Normal file
View File

Binary file not shown.

6
config.js
View File

@ -58,7 +58,7 @@ export default {
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in non-maximum suppression (0.1 means drop if overlap 10%)
scoreThreshold: 0.1, // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
},
mesh: {
enabled: true,
@ -108,9 +108,9 @@ export default {
skipFrames: 15, // how many frames to go without re-running the hand bounding box detector, only used for video inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
// as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.2, // threshold for discarding a prediction
minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.2, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score in non-maximum suppression
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on score in non-maximum suppression
enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace
maxHands: 10, // maximum number of hands detected in the input, should be set to the minimum number for performance
detector: {

10
demo/browser.js
View File

@ -286,15 +286,15 @@ async function detectSampleImages() {
function setupMenu() {
menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' });
const btn = menu.addButton('Start Video', 'Pause Video', () => detectVideo());
menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
const btn = menu.addButton('start video', 'pause video', () => detectVideo());
menu.addButton('process images', 'process images', () => detectSampleImages());
document.getElementById('play').addEventListener('click', () => btn.click());
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addList('Backend', ['cpu', 'webgl', 'wasm', 'webgpu'], human.config.backend, (val) => human.config.backend = val);
menu.addBool('Async Operations', human.config, 'async');
menu.addBool('Enable Profiler', human.config, 'profile');
menu.addBool('Memory Shield', human.config, 'deallocate');
menu.addBool('Async Operations', human.config, 'async', (val) => human.config.async = val);
menu.addBool('Enable Profiler', human.config, 'profile', (val) => human.config.profile = val);
menu.addBool('Memory Shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
menu.addBool('Use Web Worker', ui, 'useWorker');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Enabled Models');

3
demo/index.html
View File

@ -22,7 +22,8 @@
<!-- alternatively load demo sources directly -->
<!-- <script src="browser.js" type="module"></script> -->
<style>
html { font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 400; src: local('Lato'), url('../assets/lato.ttf') format('truetype'); }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden; scrollbar-width: none; }
body::-webkit-scrollbar { display: none; }
.play { position: absolute; width: 300px; height: 300px; z-index: 9; top: 30%; left: 50%; margin-left: -150px; display: none; }

5
demo/menu.js
View File

@ -32,12 +32,13 @@ function createCSS() {
.menu-list { margin-right: 0.8rem; }
select:focus { outline: none; }
.menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; }
.menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; font-weight: 800; }
.menu-chart-title { padding: 0; font-size: 0.8rem; font-weight: 800; align-items: center}
.menu-chart-canvas { background: transparent; margin: 0.2rem 0 0.2rem 0.6rem; }
.menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; border-radius: 1rem; justify-content: center; }
.menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey;
border-radius: 1rem; justify-content: center; font-family: inherit; font-variant: inherit; font-size: 1rem; font-weight: 800; }
.menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
.menu-button:focus { outline: none; }

47
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

16
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"demo/browser.js": {
"bytes": 17744,
"bytes": 17856,
"imports": [
{
"path": "dist/human.esm.js"
@ -19,11 +19,11 @@
"imports": []
},
"demo/menu.js": {
"bytes": 12460,
"bytes": 12575,
"imports": []
},
"dist/human.esm.js": {
"bytes": 1278477,
"bytes": 1278661,
"imports": []
}
},
@ -31,13 +31,13 @@
"dist/demo-browser-index.js.map": {
"imports": [],
"inputs": {},
"bytes": 5520787
"bytes": 5521435
},
"dist/demo-browser-index.js": {
"imports": [],
"inputs": {
"dist/human.esm.js": {
"bytesInOutput": 1288376
"bytesInOutput": 1288563
},
"dist/human.esm.js": {
"bytesInOutput": 0
@ -46,13 +46,13 @@
"bytesInOutput": 4708
},
"demo/menu.js": {
"bytesInOutput": 9573
"bytesInOutput": 9688
},
"demo/browser.js": {
"bytesInOutput": 11364
"bytesInOutput": 11434
}
},
"bytes": 1322785
"bytes": 1323157
}
}
}

4
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

10
dist/human.esm.json vendored
View File

@ -398,7 +398,7 @@
]
},
"src/hand/handpipeline.js": {
"bytes": 8632,
"bytes": 8839,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -433,7 +433,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13709,
"bytes": 13727,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,7 +513,7 @@
"dist/human.esm.js.map": {
"imports": [],
"inputs": {},
"bytes": 5419375
"bytes": 5419716
},
"dist/human.esm.js": {
"imports": [],
@ -654,7 +654,7 @@
"bytesInOutput": 1005
},
"src/hand/handpipeline.js": {
"bytesInOutput": 3055
"bytesInOutput": 3239
},
"src/hand/anchors.js": {
"bytesInOutput": 127001
@ -684,7 +684,7 @@
"bytesInOutput": 0
}
},
"bytes": 1278477
"bytes": 1278661
}
}
}

9
src/hand/handpipeline.js
View File

@ -88,10 +88,11 @@ class HandPipeline {
async estimateHands(image, config) {
this.skipFrames = config.skipFrames;
// don't need box detection if we have sufficient number of boxes
let useFreshBox = (this.detectedHands === 0) || (this.detectedHands !== this.regionsOfInterest.length);
let useFreshBox = (this.runsWithoutHandDetector > this.skipFrames) || (this.detectedHands !== this.regionsOfInterest.length);
console.log(this.runsWithoutHandDetector, this.skipFrames, this.detectedHands, this.regionsOfInterest.length);
let boundingBoxPredictions;
// but every skipFrames check if detect boxes number changed
if (useFreshBox || this.runsWithoutHandDetector > this.skipFrames) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
if (useFreshBox) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
// if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1
if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true;
if (useFreshBox) {
@ -144,6 +145,7 @@ class HandPipeline {
};
hands.push(result);
} else {
this.updateRegionsOfInterest(null, i);
/*
const result = {
handInViewConfidence: confidenceValue,
@ -157,6 +159,7 @@ class HandPipeline {
}
keypoints.dispose();
}
this.regionsOfInterest = this.regionsOfInterest.filter((a) => a !== null);
this.detectedHands = hands.length;
return hands;
}
@ -173,7 +176,7 @@ class HandPipeline {
updateRegionsOfInterest(newBox, i) {
const previousBox = this.regionsOfInterest[i];
let iou = 0;
if (previousBox != null && previousBox.startPoint != null) {
if (newBox && previousBox && previousBox.startPoint) {
const [boxStartX, boxStartY] = newBox.startPoint;
const [boxEndX, boxEndY] = newBox.endPoint;
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;

4
src/human.js
View File

@ -12,7 +12,7 @@ const defaults = require('../config.js').default;
const app = require('../package.json');
// static config override for non-video detection
const override = {
const disableSkipFrames = {
face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, gender: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, hand: { skipFrames: 0 },
};
@ -281,7 +281,7 @@ class Human {
// update configuration
this.config = mergeDeep(this.config, userConfig);
if (!this.config.videoOptimized) this.config = mergeDeep(this.config, override);
if (!this.config.videoOptimized) this.config = mergeDeep(this.config, disableSkipFrames);
// sanity checks
this.state = 'check';