conditional hand rotation

config.js

@@ -140,6 +140,8 @@ export default {
 
   hand: {
     enabled: true,
+    rotation: false,         // use best-guess rotated hand image or just box with rotation as-is
+                             // false means higher performance, but incorrect finger mapping if hand is inverted
     inputSize: 256,          // fixed value
     skipFrames: 19,          // how many frames to go without re-running the hand bounding box detector
                              // only used for video inputs

demo/browser.js

@@ -3,10 +3,14 @@ import draw from './draw.js';
 import Menu from './menu.js';
 import GLBench from './gl-bench.js';
 
-// const userConfig = {}; // add any user configuration overrides
+const userConfig = {}; // add any user configuration overrides
+/*
 const userConfig = {
   async: false,
+  face: { enabled: false },
+  body: { enabled: false },
 };
+*/
 
 const human = new Human(userConfig);
 
@@ -33,7 +37,7 @@ const ui = {
   console: true,
   maxFPSframes: 10,
   modelsPreload: true,
-  modelsWarmup: false,
+  modelsWarmup: true,
   menuWidth: 0,
   menuHeight: 0,
   camera: {},
@@ -44,7 +48,7 @@ const ui = {
   detectThread: null,
   framesDraw: 0,
   framesDetect: 0,
-  bench: true,
+  bench: false,
 };
 
 // global variables
@@ -471,6 +475,10 @@ function setupMenu() {
     human.config.face.detector.iouThreshold = parseFloat(val);
     human.config.hand.iouThreshold = parseFloat(val);
   });
+  menu.process.addBool('detection rotation', human.config.face.detector, 'rotation', (val) => {
+    human.config.face.detector.rotation = val;
+    human.config.hand.rotation = val;
+  });
   menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.process.addButton('process sample images', 'process images', () => detectSampleImages());
   menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');

dist/demo-browser-index.json

@@ -1,7 +1,7 @@
 {
   "inputs": {
     "dist/human.esm.js": {
-      "bytes": 1786626,
+      "bytes": 1786155,
       "imports": []
     },
     "demo/draw.js": {
@@ -17,7 +17,7 @@
       "imports": []
     },
     "demo/browser.js": {
-      "bytes": 25223,
+      "bytes": 25469,
       "imports": [
         {
           "path": "dist/human.esm.js"
@@ -38,14 +38,14 @@
     "dist/demo-browser-index.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2746890
+      "bytes": 2747800
     },
     "dist/demo-browser-index.js": {
       "imports": [],
       "exports": [],
       "inputs": {
         "dist/human.esm.js": {
-          "bytesInOutput": 1779386
+          "bytesInOutput": 1778915
         },
         "demo/draw.js": {
           "bytesInOutput": 7816
@@ -57,10 +57,10 @@
           "bytesInOutput": 7382
         },
         "demo/browser.js": {
-          "bytesInOutput": 19411
+          "bytesInOutput": 19563
         }
       },
-      "bytes": 1833184
+      "bytes": 1832865
     }
   }
 }

dist/human.esm.json

@@ -36,7 +36,7 @@
       "imports": []
     },
     "src/face/facepipeline.js": {
-      "bytes": 13817,
+      "bytes": 13856,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -279,7 +279,7 @@
       "imports": []
     },
     "src/hand/handpipeline.js": {
-      "bytes": 7607,
+      "bytes": 7923,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -289,6 +289,9 @@
         },
         {
           "path": "src/hand/util.js"
+        },
+        {
+          "path": "src/log.js"
         }
       ]
     },
@@ -297,7 +300,7 @@
       "imports": []
     },
     "src/hand/handpose.js": {
-      "bytes": 3194,
+      "bytes": 3250,
       "imports": [
         {
           "path": "src/log.js"
@@ -339,7 +342,7 @@
       ]
     },
     "config.js": {
-      "bytes": 8779,
+      "bytes": 8990,
       "imports": []
     },
     "src/sample.js": {
@@ -351,7 +354,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 16523,
+      "bytes": 16524,
       "imports": [
         {
           "path": "src/log.js"
@@ -405,7 +408,7 @@
     "dist/human.esm.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2651614
+      "bytes": 2652161
     },
     "dist/human.esm.js": {
       "imports": [],
@@ -426,7 +429,7 @@
           "bytesInOutput": 30817
         },
         "src/face/facepipeline.js": {
-          "bytesInOutput": 9297
+          "bytesInOutput": 9323
         },
         "src/face/facemesh.js": {
           "bytesInOutput": 2182
@@ -483,13 +486,13 @@
           "bytesInOutput": 2730
         },
         "src/hand/handpipeline.js": {
-          "bytesInOutput": 4624
+          "bytesInOutput": 4456
         },
         "src/hand/anchors.js": {
           "bytesInOutput": 127032
         },
         "src/hand/handpose.js": {
-          "bytesInOutput": 1953
+          "bytesInOutput": 2007
         },
         "src/gesture/gesture.js": {
           "bytesInOutput": 2463
@@ -510,13 +513,13 @@
           "bytesInOutput": 10905
         },
         "src/hand/box.js": {
-          "bytesInOutput": 1868
+          "bytesInOutput": 1473
         },
         "src/hand/util.js": {
           "bytesInOutput": 1796
         },
         "config.js": {
-          "bytesInOutput": 1428
+          "bytesInOutput": 1440
         },
         "src/sample.js": {
           "bytesInOutput": 11646
@@ -525,7 +528,7 @@
           "bytesInOutput": 21
         }
       },
-      "bytes": 1786626
+      "bytes": 1786155
     }
   }
 }

dist/human.json

@@ -36,7 +36,7 @@
       "imports": []
     },
     "src/face/facepipeline.js": {
-      "bytes": 13817,
+      "bytes": 13856,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -279,7 +279,7 @@
       "imports": []
     },
     "src/hand/handpipeline.js": {
-      "bytes": 7607,
+      "bytes": 7923,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -289,6 +289,9 @@
         },
         {
           "path": "src/hand/util.js"
+        },
+        {
+          "path": "src/log.js"
         }
       ]
     },
@@ -297,7 +300,7 @@
       "imports": []
     },
     "src/hand/handpose.js": {
-      "bytes": 3194,
+      "bytes": 3250,
       "imports": [
         {
           "path": "src/log.js"
@@ -339,7 +342,7 @@
       ]
     },
     "config.js": {
-      "bytes": 8779,
+      "bytes": 8990,
       "imports": []
     },
     "src/sample.js": {
@@ -351,7 +354,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 16523,
+      "bytes": 16524,
       "imports": [
         {
           "path": "src/log.js"
@@ -405,7 +408,7 @@
     "dist/human.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2668611
+      "bytes": 2669160
     },
     "dist/human.js": {
       "imports": [],
@@ -424,7 +427,7 @@
           "bytesInOutput": 30817
         },
         "src/face/facepipeline.js": {
-          "bytesInOutput": 9297
+          "bytesInOutput": 9323
         },
         "src/face/facemesh.js": {
           "bytesInOutput": 2182
@@ -481,13 +484,13 @@
           "bytesInOutput": 2730
         },
         "src/hand/handpipeline.js": {
-          "bytesInOutput": 4624
+          "bytesInOutput": 4456
         },
         "src/hand/anchors.js": {
           "bytesInOutput": 127032
         },
         "src/hand/handpose.js": {
-          "bytesInOutput": 1953
+          "bytesInOutput": 2007
         },
         "src/gesture/gesture.js": {
           "bytesInOutput": 2463
@@ -508,13 +511,13 @@
           "bytesInOutput": 1520210
         },
         "src/hand/box.js": {
-          "bytesInOutput": 1868
+          "bytesInOutput": 1473
         },
         "src/hand/util.js": {
           "bytesInOutput": 1796
         },
         "config.js": {
-          "bytesInOutput": 1428
+          "bytesInOutput": 1440
         },
         "src/sample.js": {
           "bytesInOutput": 11646
@@ -523,7 +526,7 @@
           "bytesInOutput": 21
         }
       },
-      "bytes": 1786700
+      "bytes": 1786229
     }
   }
 }

src/face/facepipeline.js

@@ -188,7 +188,8 @@ class Pipeline {
         face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshHeight, this.meshWidth]).div(255);
       } else {
         rotationMatrix = util.IDENTITY_MATRIX;
-        face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, input, [this.meshHeight, this.meshWidth]).div(255);
+        const cloned = input.clone();
+        face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, cloned, [this.meshHeight, this.meshWidth]).div(255);
       }
 
       // if we're not going to produce mesh, don't spend time with further processing

src/hand/handpipeline.js

@@ -18,11 +18,13 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import * as box from './box';
 import * as util from './util';
+// eslint-disable-next-line no-unused-vars
+import { log } from '../log.js';
 
-const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
+// const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
-const PALM_BOX_ENLARGE_FACTOR = 3;
+const PALM_BOX_ENLARGE_FACTOR = 5; // default 3
-const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
+// const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
-const HAND_BOX_ENLARGE_FACTOR = 1.65; // increased from model default 1.65;
+const HAND_BOX_ENLARGE_FACTOR = 1.65; // default 1.65
 const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2];
 const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
 const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
@@ -38,22 +40,20 @@ class HandPipeline {
   }
 
   getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
-    const rotatedPalmLandmarks = palmLandmarks.map((coord) => {
+    const rotatedPalmLandmarks = palmLandmarks.map((coord) => util.rotatePoint([...coord, 1], rotationMatrix));
-      const homogeneousCoordinate = [...coord, 1];
-      return util.rotatePoint(homogeneousCoordinate, rotationMatrix);
-    });
     const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks);
-    return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR);
+    // return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR);
+    return box.enlargeBox(box.squarifyBox(boxAroundPalm), PALM_BOX_ENLARGE_FACTOR);
   }
 
   getBoxForHandLandmarks(landmarks) {
     const boundingBox = this.calculateLandmarksBoundingBox(landmarks);
-    const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
+    // const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
-    const palmLandmarks = [];
+    const boxAroundHand = box.enlargeBox(box.squarifyBox(boundingBox), HAND_BOX_ENLARGE_FACTOR);
+    boxAroundHand.palmLandmarks = [];
     for (let i = 0; i < PALM_LANDMARK_IDS.length; i++) {
-      palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
+      boxAroundHand.palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
     }
-    boxAroundHand.palmLandmarks = palmLandmarks;
     return boxAroundHand;
   }
 
@@ -110,10 +110,10 @@ class HandPipeline {
       const currentBox = this.storedBoxes[i];
       if (!currentBox) continue;
       if (config.hand.landmarks) {
-        const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
+        const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]) : 0;
         const palmCenter = box.getBoxCenter(currentBox);
         const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
-        const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
+        const rotatedImage = config.hand.rotation ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone();
         const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
         const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
         const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@@ -146,7 +146,8 @@ class HandPipeline {
         }
         keypoints.dispose();
       } else {
-        const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
+        // const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
+        const enlarged = box.enlargeBox(box.squarifyBox(currentBox), HAND_BOX_ENLARGE_FACTOR);
         const result = {
           confidence: currentBox.confidence,
           box: {

src/hand/handpose.js

@@ -19,7 +19,7 @@
 import { log } from '../log.js';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as handdetector from './handdetector';
-import * as pipeline from './handpipeline';
+import * as handpipeline from './handpipeline';
 import * as anchors from './anchors';
 
 const MESH_ANNOTATIONS = {
@@ -32,8 +32,8 @@ const MESH_ANNOTATIONS = {
 };
 
 class HandPose {
-  constructor(pipe) {
+  constructor(handPipeline) {
-    this.pipeline = pipe;
+    this.handPipeline = handPipeline;
   }
 
   static getAnnotations() {
@@ -41,7 +41,7 @@ class HandPose {
   }
 
   async estimateHands(input, config) {
-    const predictions = await this.pipeline.estimateHands(input, config);
+    const predictions = await this.handPipeline.estimateHands(input, config);
     if (!predictions) return [];
     const hands = [];
     for (const prediction of predictions) {
@@ -74,11 +74,11 @@ async function load(config) {
     config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
     config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
   ]);
-  const detector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors);
+  const handDetector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors);
-  const pipe = new pipeline.HandPipeline(detector, handPoseModel, config.hand.inputSize);
+  const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, config.hand.inputSize);
-  const handpose = new HandPose(pipe);
+  const handPose = new HandPose(handPipeline);
   if (config.hand.enabled) log(`load model: ${config.hand.detector.modelPath.match(/\/(.*)\./)[1]}`);
   if (config.hand.landmarks) log(`load model: ${config.hand.skeleton.modelPath.match(/\/(.*)\./)[1]}`);
-  return handpose;
+  return handPose;
 }
 exports.load = load;

src/human.js

@@ -120,6 +120,7 @@ class Human {
       }
       this.firstRun = false;
     }
+
     if (this.config.async) {
       [
         this.models.facemesh,

wiki

@@ -1 +1 @@
-Subproject commit 60eb01217f8d3e69055c991991183dd295cf3766
+Subproject commit 640fbd9a107c52692bfaaede0d751c5572cf7f22