ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix hand detection performance

pull/280/head
Vladimir Mandic 2020-11-07 11:25:03 -05:00
parent c9649d7397
commit 50c2648711
6 changed files with 21 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.js
View File

@ -58,7 +58,7 @@ export default {
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in non-maximum suppression (0.1 means drop if overlap 10%) iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in non-maximum suppression (0.1 means drop if overlap 10%)
scoreThreshold: 0.1, // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
}, },
mesh: { mesh: {
enabled: true, enabled: true,
@ -108,9 +108,9 @@ export default {
skipFrames: 15, // how many frames to go without re-running the hand bounding box detector, only used for video inputs skipFrames: 15, // how many frames to go without re-running the hand bounding box detector, only used for video inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis // if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
// as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.2, // threshold for deciding whether boxes overlap too much in non-maximum suppression iouThreshold: 0.2, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score in non-maximum suppression scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on score in non-maximum suppression
enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace
maxHands: 10, // maximum number of hands detected in the input, should be set to the minimum number for performance maxHands: 10, // maximum number of hands detected in the input, should be set to the minimum number for performance
detector: { detector: {

10
demo/browser.js
View File

@ -286,15 +286,15 @@ async function detectSampleImages() {
function setupMenu() { function setupMenu() {
menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' }); menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' });
const btn = menu.addButton('Start Video', 'Pause Video', () => detectVideo()); const btn = menu.addButton('start video', 'pause video', () => detectVideo());
menu.addButton('Process Images', 'Process Images', () => detectSampleImages()); menu.addButton('process images', 'process images', () => detectSampleImages());
document.getElementById('play').addEventListener('click', () => btn.click()); document.getElementById('play').addEventListener('click', () => btn.click());
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">'); menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addList('Backend', ['cpu', 'webgl', 'wasm', 'webgpu'], human.config.backend, (val) => human.config.backend = val); menu.addList('Backend', ['cpu', 'webgl', 'wasm', 'webgpu'], human.config.backend, (val) => human.config.backend = val);
menu.addBool('Async Operations', human.config, 'async'); menu.addBool('Async Operations', human.config, 'async', (val) => human.config.async = val);
menu.addBool('Enable Profiler', human.config, 'profile'); menu.addBool('Enable Profiler', human.config, 'profile', (val) => human.config.profile = val);
menu.addBool('Memory Shield', human.config, 'deallocate'); menu.addBool('Memory Shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
menu.addBool('Use Web Worker', ui, 'useWorker'); menu.addBool('Use Web Worker', ui, 'useWorker');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">'); menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Enabled Models'); menu.addLabel('Enabled Models');

3
demo/index.html
View File

@ -22,7 +22,8 @@
<!-- alternatively load demo sources directly --> <!-- alternatively load demo sources directly -->
<!-- <script src="browser.js" type="module"></script> --> <!-- <script src="browser.js" type="module"></script> -->
<style> <style>
html { font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; } @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 400; src: local('Lato'), url('../assets/lato.ttf') format('truetype'); }
html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
body { margin: 0; background: black; color: white; overflow-x: hidden; scrollbar-width: none; } body { margin: 0; background: black; color: white; overflow-x: hidden; scrollbar-width: none; }
body::-webkit-scrollbar { display: none; } body::-webkit-scrollbar { display: none; }
.play { position: absolute; width: 300px; height: 300px; z-index: 9; top: 30%; left: 50%; margin-left: -150px; display: none; } .play { position: absolute; width: 300px; height: 300px; z-index: 9; top: 30%; left: 50%; margin-left: -150px; display: none; }

5
demo/menu.js
View File

@ -32,12 +32,13 @@ function createCSS() {
.menu-list { margin-right: 0.8rem; } .menu-list { margin-right: 0.8rem; }
select:focus { outline: none; } select:focus { outline: none; }
.menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; } .menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; font-weight: 800; }
.menu-chart-title { padding: 0; font-size: 0.8rem; font-weight: 800; align-items: center} .menu-chart-title { padding: 0; font-size: 0.8rem; font-weight: 800; align-items: center}
.menu-chart-canvas { background: transparent; margin: 0.2rem 0 0.2rem 0.6rem; } .menu-chart-canvas { background: transparent; margin: 0.2rem 0 0.2rem 0.6rem; }
.menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; border-radius: 1rem; justify-content: center; } .menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey;
border-radius: 1rem; justify-content: center; font-family: inherit; font-variant: inherit; font-size: 1rem; font-weight: 800; }
.menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; } .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
.menu-button:focus { outline: none; } .menu-button:focus { outline: none; }

9
src/hand/handpipeline.js
View File

@ -88,10 +88,11 @@ class HandPipeline {
async estimateHands(image, config) { async estimateHands(image, config) {
this.skipFrames = config.skipFrames; this.skipFrames = config.skipFrames;
// don't need box detection if we have sufficient number of boxes // don't need box detection if we have sufficient number of boxes
let useFreshBox = (this.detectedHands === 0) || (this.detectedHands !== this.regionsOfInterest.length); let useFreshBox = (this.runsWithoutHandDetector > this.skipFrames) || (this.detectedHands !== this.regionsOfInterest.length);
console.log(this.runsWithoutHandDetector, this.skipFrames, this.detectedHands, this.regionsOfInterest.length);
let boundingBoxPredictions; let boundingBoxPredictions;
// but every skipFrames check if detect boxes number changed // but every skipFrames check if detect boxes number changed
if (useFreshBox || this.runsWithoutHandDetector > this.skipFrames) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config); if (useFreshBox) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
// if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1 // if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1
if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true; if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true;
if (useFreshBox) { if (useFreshBox) {
@ -144,6 +145,7 @@ class HandPipeline {
}; };
hands.push(result); hands.push(result);
} else { } else {
this.updateRegionsOfInterest(null, i);
/* /*
const result = { const result = {
handInViewConfidence: confidenceValue, handInViewConfidence: confidenceValue,
@ -157,6 +159,7 @@ class HandPipeline {
} }
keypoints.dispose(); keypoints.dispose();
} }
this.regionsOfInterest = this.regionsOfInterest.filter((a) => a !== null);
this.detectedHands = hands.length; this.detectedHands = hands.length;
return hands; return hands;
} }
@ -173,7 +176,7 @@ class HandPipeline {
updateRegionsOfInterest(newBox, i) { updateRegionsOfInterest(newBox, i) {
const previousBox = this.regionsOfInterest[i]; const previousBox = this.regionsOfInterest[i];
let iou = 0; let iou = 0;
if (previousBox != null && previousBox.startPoint != null) { if (newBox && previousBox && previousBox.startPoint) {
const [boxStartX, boxStartY] = newBox.startPoint; const [boxStartX, boxStartY] = newBox.startPoint;
const [boxEndX, boxEndY] = newBox.endPoint; const [boxEndX, boxEndY] = newBox.endPoint;
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint; const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;

4
src/human.js
View File

@ -12,7 +12,7 @@ const defaults = require('../config.js').default;
const app = require('../package.json'); const app = require('../package.json');
// static config override for non-video detection // static config override for non-video detection
const override = { const disableSkipFrames = {
face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, gender: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, hand: { skipFrames: 0 }, face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, gender: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, hand: { skipFrames: 0 },
}; };
@ -281,7 +281,7 @@ class Human {
// update configuration // update configuration
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
if (!this.config.videoOptimized) this.config = mergeDeep(this.config, override); if (!this.config.videoOptimized) this.config = mergeDeep(this.config, disableSkipFrames);
// sanity checks // sanity checks
this.state = 'check'; this.state = 'check';