fix hand detection performance

config.js

@@ -58,7 +58,7 @@ export default {
                              // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
       minConfidence: 0.1,    // threshold for discarding a prediction
       iouThreshold: 0.1,     // threshold for deciding whether boxes overlap too much in non-maximum suppression (0.1 means drop if overlap 10%)
-      scoreThreshold: 0.1,   // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
+      scoreThreshold: 0.2,   // threshold for deciding when to remove boxes based on score in non-maximum suppression, this is applied on detection objects only and before minConfidence
     },
     mesh: {
       enabled: true,
@@ -108,9 +108,9 @@ export default {
     skipFrames: 15,          // how many frames to go without re-running the hand bounding box detector, only used for video inputs
                              // if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
                              // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
-    minConfidence: 0.2,      // threshold for discarding a prediction
+    minConfidence: 0.5,      // threshold for discarding a prediction
     iouThreshold: 0.2,       // threshold for deciding whether boxes overlap too much in non-maximum suppression
-    scoreThreshold: 0.2,     // threshold for deciding when to remove boxes based on score in non-maximum suppression
+    scoreThreshold: 0.5,     // threshold for deciding when to remove boxes based on score in non-maximum suppression
     enlargeFactor: 1.65,     // empiric tuning as skeleton prediction prefers hand box with some whitespace
     maxHands: 10,            // maximum number of hands detected in the input, should be set to the minimum number for performance
     detector: {

demo/browser.js

@@ -286,15 +286,15 @@ async function detectSampleImages() {
 
 function setupMenu() {
   menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' });
-  const btn = menu.addButton('Start Video', 'Pause Video', () => detectVideo());
+  const btn = menu.addButton('start video', 'pause video', () => detectVideo());
-  menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
+  menu.addButton('process images', 'process images', () => detectSampleImages());
   document.getElementById('play').addEventListener('click', () => btn.click());
 
   menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
   menu.addList('Backend', ['cpu', 'webgl', 'wasm', 'webgpu'], human.config.backend, (val) => human.config.backend = val);
-  menu.addBool('Async Operations', human.config, 'async');
+  menu.addBool('Async Operations', human.config, 'async', (val) => human.config.async = val);
-  menu.addBool('Enable Profiler', human.config, 'profile');
+  menu.addBool('Enable Profiler', human.config, 'profile', (val) => human.config.profile = val);
-  menu.addBool('Memory Shield', human.config, 'deallocate');
+  menu.addBool('Memory Shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
   menu.addBool('Use Web Worker', ui, 'useWorker');
   menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
   menu.addLabel('Enabled Models');

demo/index.html

@@ -22,7 +22,8 @@
     <!-- alternatively load demo sources directly -->
     <!-- <script src="browser.js" type="module"></script> -->
     <style>
-      html { font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
+      @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 400; src: local('Lato'), url('../assets/lato.ttf') format('truetype'); }
+      html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
       body { margin: 0; background: black; color: white; overflow-x: hidden; scrollbar-width: none; }
       body::-webkit-scrollbar { display: none; }
       .play { position: absolute; width: 300px; height: 300px; z-index: 9; top: 30%; left: 50%; margin-left: -150px; display: none; }

demo/menu.js

@@ -32,12 +32,13 @@ function createCSS() {
 
   .menu-list { margin-right: 0.8rem; }
   select:focus { outline: none; }
-  .menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; }
+  .menu-list-item { background: ${theme.itemBackground}; color: ${theme.itemColor}; border: none; padding: 0.2rem; font-family: inherit; font-variant: inherit; border-radius: 1rem; font-weight: 800; }
 
   .menu-chart-title { padding: 0; font-size: 0.8rem; font-weight: 800; align-items: center}
   .menu-chart-canvas { background: transparent; margin: 0.2rem 0 0.2rem 0.6rem; }
   
-  .menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; border-radius: 1rem; justify-content: center; }
+  .menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey;
+                 border-radius: 1rem; justify-content: center; font-family: inherit; font-variant: inherit; font-size: 1rem; font-weight: 800; }
   .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; }
   .menu-button:focus { outline: none; }
 

dist/demo-browser-index.json

@@ -1,7 +1,7 @@
 {
   "inputs": {
     "demo/browser.js": {
-      "bytes": 17744,
+      "bytes": 17856,
       "imports": [
         {
           "path": "dist/human.esm.js"
@@ -19,11 +19,11 @@
       "imports": []
     },
     "demo/menu.js": {
-      "bytes": 12460,
+      "bytes": 12575,
       "imports": []
     },
     "dist/human.esm.js": {
-      "bytes": 1278477,
+      "bytes": 1278661,
       "imports": []
     }
   },
@@ -31,13 +31,13 @@
     "dist/demo-browser-index.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5520787
+      "bytes": 5521435
     },
     "dist/demo-browser-index.js": {
       "imports": [],
       "inputs": {
         "dist/human.esm.js": {
-          "bytesInOutput": 1288376
+          "bytesInOutput": 1288563
         },
         "dist/human.esm.js": {
           "bytesInOutput": 0
@@ -46,13 +46,13 @@
           "bytesInOutput": 4708
         },
         "demo/menu.js": {
-          "bytesInOutput": 9573
+          "bytesInOutput": 9688
         },
         "demo/browser.js": {
-          "bytesInOutput": 11364
+          "bytesInOutput": 11434
         }
       },
-      "bytes": 1322785
+      "bytes": 1323157
     }
   }
 }

dist/human.esm.json

@@ -398,7 +398,7 @@
       ]
     },
     "src/hand/handpipeline.js": {
-      "bytes": 8632,
+      "bytes": 8839,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -433,7 +433,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 13709,
+      "bytes": 13727,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -513,7 +513,7 @@
     "dist/human.esm.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5419375
+      "bytes": 5419716
     },
     "dist/human.esm.js": {
       "imports": [],
@@ -654,7 +654,7 @@
           "bytesInOutput": 1005
         },
         "src/hand/handpipeline.js": {
-          "bytesInOutput": 3055
+          "bytesInOutput": 3239
         },
         "src/hand/anchors.js": {
           "bytesInOutput": 127001
@@ -684,7 +684,7 @@
           "bytesInOutput": 0
         }
       },
-      "bytes": 1278477
+      "bytes": 1278661
     }
   }
 }

src/hand/handpipeline.js

@@ -88,10 +88,11 @@ class HandPipeline {
   async estimateHands(image, config) {
     this.skipFrames = config.skipFrames;
     // don't need box detection if we have sufficient number of boxes
-    let useFreshBox = (this.detectedHands === 0) || (this.detectedHands !== this.regionsOfInterest.length);
+    let useFreshBox = (this.runsWithoutHandDetector > this.skipFrames) || (this.detectedHands !== this.regionsOfInterest.length);
+    console.log(this.runsWithoutHandDetector, this.skipFrames, this.detectedHands, this.regionsOfInterest.length);
     let boundingBoxPredictions;
     // but every skipFrames check if detect boxes number changed
-    if (useFreshBox || this.runsWithoutHandDetector > this.skipFrames) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
+    if (useFreshBox) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
     // if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1
     if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true;
     if (useFreshBox) {
@@ -144,6 +145,7 @@ class HandPipeline {
         };
         hands.push(result);
       } else {
+        this.updateRegionsOfInterest(null, i);
         /*
         const result = {
           handInViewConfidence: confidenceValue,
@@ -157,6 +159,7 @@ class HandPipeline {
       }
       keypoints.dispose();
     }
+    this.regionsOfInterest = this.regionsOfInterest.filter((a) => a !== null);
     this.detectedHands = hands.length;
     return hands;
   }
@@ -173,7 +176,7 @@ class HandPipeline {
   updateRegionsOfInterest(newBox, i) {
     const previousBox = this.regionsOfInterest[i];
     let iou = 0;
-    if (previousBox != null && previousBox.startPoint != null) {
+    if (newBox && previousBox && previousBox.startPoint) {
       const [boxStartX, boxStartY] = newBox.startPoint;
       const [boxEndX, boxEndY] = newBox.endPoint;
       const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;

src/human.js

@@ -12,7 +12,7 @@ const defaults = require('../config.js').default;
 const app = require('../package.json');
 
 // static config override for non-video detection
-const override = {
+const disableSkipFrames = {
   face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, gender: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, hand: { skipFrames: 0 },
 };
 
@@ -281,7 +281,7 @@ class Human {
 
     // update configuration
     this.config = mergeDeep(this.config, userConfig);
-    if (!this.config.videoOptimized) this.config = mergeDeep(this.config, override);
+    if (!this.config.videoOptimized) this.config = mergeDeep(this.config, disableSkipFrames);
 
     // sanity checks
     this.state = 'check';