update handtrack boxes and refactor handpose

CHANGELOG.md

@@ -9,11 +9,12 @@
   
 ## Changelog
   
+### **HEAD -> main** 2021/10/19 mandic00@live.com
+
+
 ### **2.3.5** 2021/10/19 mandic00@live.com
 
-
+- removed direct usage of performance.now
-### **origin/main** 2021/10/19 snaggadhagga@gmail.com
-
 
 ### **2.3.4** 2021/10/19 mandic00@live.com
 

package.json

@@ -69,7 +69,7 @@
     "@types/node": "^16.11.1",
     "@typescript-eslint/eslint-plugin": "^5.1.0",
     "@typescript-eslint/parser": "^5.1.0",
-    "@vladmandic/build": "^0.6.1",
+    "@vladmandic/build": "^0.6.2",
     "@vladmandic/pilogger": "^0.3.3",
     "canvas": "^2.8.0",
     "dayjs": "^1.10.7",

src/hand/handpose.ts

@@ -6,9 +6,9 @@
 
 import { log, join } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as handdetector from './handdetector';
+import * as handdetector from './handposedetector';
-import * as handpipeline from './handpipeline';
+import * as handpipeline from './handposepipeline';
-import * as fingerPose from '../hand/fingerpose';
+import * as fingerPose from './fingerpose';
 import type { HandResult, Box, Point } from '../result';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import type { Config } from '../config';

src/hand/handposedetector.ts

@@ -4,8 +4,8 @@
  */
 
 import * as tf from '../../dist/tfjs.esm.js';
-import * as box from './box';
+import * as util from './handposeutil';
-import * as anchors from './anchors';
+import * as anchors from './handposeanchors';
 import type { Tensor, GraphModel } from '../tfjs/types';
 
 export class HandDetector {
@@ -81,7 +81,7 @@ export class HandDetector {
       const palmLandmarks = await prediction.palmLandmarks.array();
       tf.dispose(prediction.box);
       tf.dispose(prediction.palmLandmarks);
-      hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
+      hands.push(util.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
     }
     return hands;
   }

src/hand/handposepipeline.ts

@@ -4,9 +4,8 @@
  */
 
 import * as tf from '../../dist/tfjs.esm.js';
-import * as box from './box';
+import * as util from './handposeutil';
-import * as util from './util';
+import type * as detector from './handposedetector';
-import type * as detector from './handdetector';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import { env } from '../util/env';
 
@@ -45,12 +44,12 @@ export class HandPipeline {
   getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
     const rotatedPalmLandmarks = palmLandmarks.map((coord) => util.rotatePoint([...coord, 1], rotationMatrix));
     const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks);
-    return box.enlargeBox(box.squarifyBox(boxAroundPalm), palmBoxEnlargeFactor);
+    return util.enlargeBox(util.squarifyBox(boxAroundPalm), palmBoxEnlargeFactor);
   }
 
   getBoxForHandLandmarks(landmarks) {
     const boundingBox = this.calculateLandmarksBoundingBox(landmarks);
-    const boxAroundHand = box.enlargeBox(box.squarifyBox(boundingBox), handBoxEnlargeFactor);
+    const boxAroundHand = util.enlargeBox(util.squarifyBox(boundingBox), handBoxEnlargeFactor);
     boxAroundHand.palmLandmarks = [];
     for (let i = 0; i < palmLandmarkIds.length; i++) {
       boxAroundHand.palmLandmarks.push(landmarks[palmLandmarkIds[i]].slice(0, 2));
@@ -59,7 +58,7 @@ export class HandPipeline {
   }
 
   transformRawCoords(rawCoords, box2, angle, rotationMatrix) {
-    const boxSize = box.getBoxSize(box2);
+    const boxSize = util.getBoxSize(box2);
     const scaleFactor = [boxSize[0] / this.inputSize, boxSize[1] / this.inputSize, (boxSize[0] + boxSize[1]) / this.inputSize / 2];
     const coordsScaled = rawCoords.map((coord) => [
       scaleFactor[0] * (coord[0] - this.inputSize / 2),
@@ -72,7 +71,7 @@ export class HandPipeline {
       return [...rotated, coord[2]];
     });
     const inverseRotationMatrix = util.invertTransformMatrix(rotationMatrix);
-    const boxCenter = [...box.getBoxCenter(box2), 1];
+    const boxCenter = [...util.getBoxCenter(box2), 1];
     const originalBoxCenter = [
       util.dot(boxCenter, inverseRotationMatrix[0]),
       util.dot(boxCenter, inverseRotationMatrix[1]),
@@ -112,12 +111,12 @@ export class HandPipeline {
       if (!currentBox) continue;
       if (config.hand.landmarks) {
         const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
-        const palmCenter = box.getBoxCenter(currentBox);
+        const palmCenter = util.getBoxCenter(currentBox);
         const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
         const rotatedImage = config.hand.rotation && env.kernels.includes('rotatewithoffset') ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone();
         const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
         const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
-        const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
+        const croppedInput = util.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
         const handImage = tf.div(croppedInput, 255);
         tf.dispose(croppedInput);
         tf.dispose(rotatedImage);
@@ -148,7 +147,7 @@ export class HandPipeline {
         tf.dispose(keypoints);
       } else {
         // const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), handBoxEnlargeFactor);
-        const enlarged = box.enlargeBox(box.squarifyBox(currentBox), handBoxEnlargeFactor);
+        const enlarged = util.enlargeBox(util.squarifyBox(currentBox), handBoxEnlargeFactor);
         const result = {
           confidence: currentBox.confidence,
           boxConfidence: currentBox.confidence,

src/hand/handposeutil.ts

@@ -1,8 +1,3 @@
-/**
- * HandPose model implementation
- * See `handpose.ts` for entry point
- */
-
 import * as tf from '../../dist/tfjs.esm.js';
 
 export function getBoxSize(box) {
@@ -70,3 +65,73 @@ export function shiftBox(box, shiftFactor) {
   const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]];
   return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
 }
+
+export function normalizeRadians(angle) {
+  return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI));
+}
+
+export function computeRotation(point1, point2) {
+  const radians = Math.PI / 2 - Math.atan2(-(point2[1] - point1[1]), point2[0] - point1[0]);
+  return normalizeRadians(radians);
+}
+
+export const buildTranslationMatrix = (x, y) => [[1, 0, x], [0, 1, y], [0, 0, 1]];
+
+export function dot(v1, v2) {
+  let product = 0;
+  for (let i = 0; i < v1.length; i++) {
+    product += v1[i] * v2[i];
+  }
+  return product;
+}
+
+export function getColumnFrom2DArr(arr, columnIndex) {
+  const column: Array<number> = [];
+  for (let i = 0; i < arr.length; i++) {
+    column.push(arr[i][columnIndex]);
+  }
+  return column;
+}
+
+export function multiplyTransformMatrices(mat1, mat2) {
+  const product: Array<number[]> = [];
+  const size = mat1.length;
+  for (let row = 0; row < size; row++) {
+    product.push([]);
+    for (let col = 0; col < size; col++) {
+      product[row].push(dot(mat1[row], getColumnFrom2DArr(mat2, col)));
+    }
+  }
+  return product;
+}
+
+export function buildRotationMatrix(rotation, center) {
+  const cosA = Math.cos(rotation);
+  const sinA = Math.sin(rotation);
+  const rotationMatrix = [[cosA, -sinA, 0], [sinA, cosA, 0], [0, 0, 1]];
+  const translationMatrix = buildTranslationMatrix(center[0], center[1]);
+  const translationTimesRotation = multiplyTransformMatrices(translationMatrix, rotationMatrix);
+  const negativeTranslationMatrix = buildTranslationMatrix(-center[0], -center[1]);
+  return multiplyTransformMatrices(translationTimesRotation, negativeTranslationMatrix);
+}
+
+export function invertTransformMatrix(matrix) {
+  const rotationComponent = [[matrix[0][0], matrix[1][0]], [matrix[0][1], matrix[1][1]]];
+  const translationComponent = [matrix[0][2], matrix[1][2]];
+  const invertedTranslation = [
+    -dot(rotationComponent[0], translationComponent),
+    -dot(rotationComponent[1], translationComponent),
+  ];
+  return [
+    rotationComponent[0].concat(invertedTranslation[0]),
+    rotationComponent[1].concat(invertedTranslation[1]),
+    [0, 0, 1],
+  ];
+}
+
+export function rotatePoint(homogeneousCoordinate, rotationMatrix) {
+  return [
+    dot(homogeneousCoordinate, rotationMatrix[0]),
+    dot(homogeneousCoordinate, rotationMatrix[1]),
+  ];
+}

src/hand/handtrack.ts

@@ -219,6 +219,11 @@ export async function predict(input: Tensor, config: Config): Promise<HandResult
         }
       }
     }
+    for (let i = 0; i < cache.hands.length; i++) { // replace deteced boxes with calculated boxes in final output
+      const bbox = box.calc(cache.hands[i].keypoints, outputSize);
+      cache.hands[i].box = bbox.box;
+      cache.hands[i].boxRaw = bbox.boxRaw;
+    }
     resolve(cache.hands);
   });
 }

src/human.ts

@@ -17,7 +17,7 @@ import * as face from './face/face';
 import * as facemesh from './face/facemesh';
 import * as faceres from './face/faceres';
 import * as gesture from './gesture/gesture';
-import * as handpose from './handpose/handpose';
+import * as handpose from './hand/handpose';
 import * as handtrack from './hand/handtrack';
 import * as humangl from './tfjs/humangl';
 import * as image from './image/image';

src/models.ts

@@ -13,7 +13,7 @@ import * as efficientpose from './body/efficientpose';
 import * as emotion from './gear/emotion';
 import * as facemesh from './face/facemesh';
 import * as faceres from './face/faceres';
-import * as handpose from './handpose/handpose';
+import * as handpose from './hand/handpose';
 import * as handtrack from './hand/handtrack';
 import * as iris from './face/iris';
 import * as movenet from './body/movenet';