From 250187f259654b5378267fc39c376618f1b37057 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 9 Nov 2020 14:26:10 -0500
Subject: [PATCH] reimplemented blazeface processing

---
 config.js                |  6 +++---
 demo/browser.js          |  3 +--
 demo/index.html          |  4 ++--
 src/gesture.js           | 24 +++++++++++++-----------
 src/hand/handpipeline.js |  5 +++--
 src/human.js             |  6 ++++++
 src/image.js             |  4 +++-
 7 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/config.js b/config.js
index c806503c..bd761720 100644
--- a/config.js
+++ b/config.js
@@ -71,10 +71,10 @@ export default {
                              // e.g., if model is running st 25 FPS, we can re-use existing bounding
                              // box for updated face analysis as the head probably hasn't moved much
                              // in short time (10 * 1/25 = 0.25 sec)
-      minConfidence: 0.1,    // threshold for discarding a prediction
-      iouThreshold: 0.1,     // threshold for deciding whether boxes overlap too much in
+      minConfidence: 0.5,    // threshold for discarding a prediction
+      iouThreshold: 0.2,     // threshold for deciding whether boxes overlap too much in
                              // non-maximum suppression (0.1 means drop if overlap 10%)
-      scoreThreshold: 0.2,   // threshold for deciding when to remove boxes based on score
+      scoreThreshold: 0.5,   // threshold for deciding when to remove boxes based on score
                              // in non-maximum suppression,
                              // this is applied on detection objects only and before minConfidence
     },
diff --git a/demo/browser.js b/demo/browser.js
index 598b5d94..e47942ea 100644
--- a/demo/browser.js
+++ b/demo/browser.js
@@ -393,8 +393,7 @@ async function main() {
   // this is not required, just pre-warms all models for faster initial inference
   if (ui.modelsWarmup) {
     status('initializing');
-    const warmup = new ImageData(256, 256);
-    await human.detect(warmup, userConfig);
+    await human.warmup(userConfig);
   }
   status('human: ready');
   document.getElementById('loader').style.display = 'none';
diff --git a/demo/index.html b/demo/index.html
index b45f2bf5..6f798dd5 100644
--- a/demo/index.html
+++ b/demo/index.html
@@ -19,8 +19,8 @@
     <!-- <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgpu@0.0.1-alpha.0/dist/tf-webgpu.js"></script> -->
     <!-- load compiled demo js -->
     <script src="../dist/demo-browser-index.js"></script>
-    <!-- alternatively load demo sources directly -->
-    <!-- <script src="browser.js" type="module"></script> -->
+    <!-- alternatively load demo sources directly, this is not supported on mobile platforms as they don't support type=module -->
+    <!-- <script src="./browser.js" type="module"></script> -->
     <style>
       @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 400; src: local('Lato'), url('../assets/lato.ttf') format('truetype'); }
       html { font-family: 'Lato', 'Segoe UI'; font-size: 16px; font-variant: small-caps; }
diff --git a/src/gesture.js b/src/gesture.js
index 228b44c4..1fa3bde1 100644
--- a/src/gesture.js
+++ b/src/gesture.js
@@ -25,17 +25,19 @@ exports.face = (res) => {
     // if (face.annotations['rightCheek'] && face.annotations['leftCheek'] && (face.annotations['rightCheek'].length > 0) && (face.annotations['leftCheek'].length > 0)) {
     //  gestures.push(`facing ${((face.annotations['rightCheek'][0][2] > 0) || (face.annotations['leftCheek'][0][2] < 0)) ? 'right' : 'left'}`);
     // }
-    const eyeFacing = face.mesh[35][2] - face.mesh[263][2];
-    if (Math.abs(eyeFacing) < 10) gestures.push('facing camera');
-    else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`);
-    const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
-    if (openLeft < 0.2) gestures.push('blink left eye');
-    const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
-    if (openRight < 0.2) gestures.push('blink right eye');
-    const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1]));
-    if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`);
-    const chinDepth = face.mesh[152][2];
-    if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`);
+    if (face.mesh && face.mesh.length > 0) {
+      const eyeFacing = face.mesh[35][2] - face.mesh[263][2];
+      if (Math.abs(eyeFacing) < 10) gestures.push('facing camera');
+      else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`);
+      const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
+      if (openLeft < 0.2) gestures.push('blink left eye');
+      const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
+      if (openRight < 0.2) gestures.push('blink right eye');
+      const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1]));
+      if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`);
+      const chinDepth = face.mesh[152][2];
+      if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`);
+    }
   }
   return gestures;
 };
diff --git a/src/hand/handpipeline.js b/src/hand/handpipeline.js
index 0439eecb..93a80406 100644
--- a/src/hand/handpipeline.js
+++ b/src/hand/handpipeline.js
@@ -33,7 +33,7 @@ class HandPipeline {
     this.meshDetector = meshDetector;
     this.inputSize = inputSize;
     this.storedBoxes = [];
-    this.skipped = 0;
+    this.skipped = 1000;
     this.detectedHands = 0;
   }
 
@@ -91,7 +91,8 @@ class HandPipeline {
     let boxes;
     if ((this.skipped > config.skipFrames) || !config.landmarks) {
       boxes = await this.boxDetector.estimateHandBounds(image, config);
-      this.skipped = 0;
+      // don't reset on test image
+      if ((image.shape[1] !== 255) && (image.shape[2] !== 255)) this.skipped = 0;
     }
 
     // if detector result count doesn't match current working set, use it to reset current working set
diff --git a/src/human.js b/src/human.js
index c0984976..396d8425 100644
--- a/src/human.js
+++ b/src/human.js
@@ -373,6 +373,12 @@ class Human {
       resolve({ face: faceRes, body: poseRes, hand: handRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas });
     });
   }
+
+  async warmup(userConfig) {
+    const warmup = new ImageData(255, 255);
+    await this.detect(warmup, userConfig);
+    this.log('warmed up');
+  }
 }
 
 export { Human as default };
diff --git a/src/image.js b/src/image.js
index fe64cd31..cabc66f9 100644
--- a/src/image.js
+++ b/src/image.js
@@ -52,8 +52,10 @@ function process(input, config) {
       if (config.filter.polaroid) this.fx.addFilter('polaroid');
       if (config.filter.pixelate !== 0) this.fx.addFilter('pixelate', config.filter.pixelate);
       this.fx.apply(inCanvas);
+    } else {
+      outCanvas = inCanvas;
     }
-    if (!outCanvas) outCanvas = inCanvas;
+    // if (!outCanvas) outCanvas = inCanvas;
     let pixels;
     if ((config.backend === 'webgl') || (outCanvas instanceof ImageData)) {
       // tf kernel-optimized method to get imagedata, also if input is imagedata, just use it