update types and dependencies

CHANGELOG.md

@@ -11,6 +11,7 @@
   
 ### **HEAD -> main** 2021/09/27 mandic00@live.com
 
+- define app specific types
 - implement box caching for movenet
 - autodetect number of bodies and hands
 - upload new samples

package.json

@@ -67,8 +67,8 @@
     "@tensorflow/tfjs-node": "^3.9.0",
     "@tensorflow/tfjs-node-gpu": "^3.9.0",
     "@types/node": "^16.10.1",
-    "@typescript-eslint/eslint-plugin": "^4.31.2",
-    "@typescript-eslint/parser": "^4.31.2",
+    "@typescript-eslint/eslint-plugin": "^4.32.0",
+    "@typescript-eslint/parser": "^4.32.0",
     "@vladmandic/build": "^0.5.3",
     "@vladmandic/pilogger": "^0.3.3",
     "canvas": "^2.8.0",

src/body/efficientpose.ts

@@ -6,14 +6,14 @@
 
 import { log, join } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
-import type { BodyResult, Box } from '../result';
+import type { BodyResult, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
 
-type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
+type Keypoints = { score: number, part: string, position: Point, positionRaw: Point };
 
 const keypoints: Array<Keypoints> = [];
 let box: Box = [0, 0, 0, 0];

src/body/movenet.ts

@@ -45,7 +45,7 @@ async function parseSinglePose(res, config, image, inputBox) {
   for (let id = 0; id < kpt.length; id++) {
     score = kpt[id][2];
     if (score > config.body.minConfidence) {
-      const positionRaw: [number, number] = [
+      const positionRaw: Point = [
         (inputBox[3] - inputBox[1]) * kpt[id][1] + inputBox[1],
         (inputBox[2] - inputBox[0]) * kpt[id][0] + inputBox[0],
       ];
@@ -93,7 +93,7 @@ async function parseMultiPose(res, config, image, inputBox) {
     for (let i = 0; i < 17; i++) {
       const partScore = Math.round(100 * kpt[3 * i + 2]) / 100;
       if (partScore > config.body.minConfidence) {
-        const positionRaw: [number, number] = [
+        const positionRaw: Point = [
           (inputBox[3] - inputBox[1]) * kpt[3 * i + 1] + inputBox[1],
           (inputBox[2] - inputBox[0]) * kpt[3 * i + 0] + inputBox[0],
         ];

src/hand/handtrack.ts

@@ -9,7 +9,7 @@
 import { log, join } from '../util/util';
 import { scale } from '../util/box';
 import * as tf from '../../dist/tfjs.esm.js';
-import type { HandResult, Box } from '../result';
+import type { HandResult, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
@@ -25,7 +25,7 @@ const inputSize = [[0, 0], [0, 0]];
 const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip'];
 
 let skipped = 0;
-let outputSize: [number, number] = [0, 0];
+let outputSize: Point = [0, 0];
 
 type HandDetectResult = {
   id: number,
@@ -115,7 +115,7 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
       let yxBox: Box = [0, 0, 0, 0];
       if (config.hand.landmarks) { // scale box
         const detectedBox: Box = await boxSlice.data();
-        const boxCenter: [number, number] = [(detectedBox[0] + detectedBox[2]) / 2, (detectedBox[1] + detectedBox[3]) / 2];
+        const boxCenter: Point = [(detectedBox[0] + detectedBox[2]) / 2, (detectedBox[1] + detectedBox[3]) / 2];
         const boxDiff: Box = [+boxCenter[0] - detectedBox[0], +boxCenter[1] - detectedBox[1], -boxCenter[0] + detectedBox[2], -boxCenter[1] + detectedBox[3]];
         yxBox = [boxCenter[0] - boxScaleFact * boxDiff[0], boxCenter[1] - boxScaleFact * boxDiff[1], boxCenter[0] + boxScaleFact * boxDiff[2], boxCenter[1] + boxScaleFact * boxDiff[3]];
       } else { // use box as-is
@@ -163,11 +163,11 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
     if (score > (config.hand.minConfidence || 0)) {
       hand.fingerScore = score;
       t.reshaped = tf.reshape(t.keypoints, [-1, 3]);
-      const rawCoords = await t.reshaped.array() as number[];
-      hand.keypoints = (rawCoords as number[]).map((coord) => [
+      const rawCoords = await t.reshaped.array() as Point[];
+      hand.keypoints = (rawCoords as Point[]).map((coord) => [
         (h.box[2] * coord[0] / inputSize[1][0]) + h.box[0],
         (h.box[3] * coord[1] / inputSize[1][1]) + h.box[1],
-        (h.box[2] + h.box[3]) / 2 / inputSize[1][0] * coord[2],
+        (h.box[2] + h.box[3]) / 2 / inputSize[1][0] * (coord[2] || 0),
       ]);
       const updatedBox = scale(hand.keypoints, boxScaleFact, outputSize); // replace detected box with box calculated around keypoints
       h.box = updatedBox.box;

src/tfjs/backend.ts

@@ -77,6 +77,7 @@ export async function check(instance, force = false) {
       tf.ENV.set('WEBGL_CPU_FORWARD', true);
       tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', false);
       tf.ENV.set('WEBGL_USE_SHAPES_UNIFORMS', true);
+      tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 128);
       // if (!instance.config.object.enabled) tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // safe to use 16bit precision
       if (typeof instance.config['deallocate'] !== 'undefined' && instance.config['deallocate']) { // hidden param
         log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);