From b6f7d683e1f4571a24956eb6f23f57b4e412a6e5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 27 Sep 2021 14:39:54 -0400 Subject: [PATCH] update types and dependencies --- CHANGELOG.md | 1 + package.json | 4 ++-- src/body/efficientpose.ts | 4 ++-- src/body/movenet.ts | 4 ++-- src/hand/handtrack.ts | 12 ++++++------ src/tfjs/backend.ts | 1 + 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba1d415..23d6b9e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### **HEAD -> main** 2021/09/27 mandic00@live.com +- define app specific types - implement box caching for movenet - autodetect number of bodies and hands - upload new samples diff --git a/package.json b/package.json index 6cb4ecbc..51e24130 100644 --- a/package.json +++ b/package.json @@ -67,8 +67,8 @@ "@tensorflow/tfjs-node": "^3.9.0", "@tensorflow/tfjs-node-gpu": "^3.9.0", "@types/node": "^16.10.1", - "@typescript-eslint/eslint-plugin": "^4.31.2", - "@typescript-eslint/parser": "^4.31.2", + "@typescript-eslint/eslint-plugin": "^4.32.0", + "@typescript-eslint/parser": "^4.32.0", "@vladmandic/build": "^0.5.3", "@vladmandic/pilogger": "^0.3.3", "canvas": "^2.8.0", diff --git a/src/body/efficientpose.ts b/src/body/efficientpose.ts index 7b4a5ec4..e6eff298 100644 --- a/src/body/efficientpose.ts +++ b/src/body/efficientpose.ts @@ -6,14 +6,14 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; -import type { BodyResult, Box } from '../result'; +import type { BodyResult, Box, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; let model: GraphModel | null; -type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] }; +type Keypoints = { score: number, part: string, position: Point, positionRaw: Point }; const keypoints: Array = []; let box: Box = [0, 0, 0, 0]; diff --git a/src/body/movenet.ts b/src/body/movenet.ts index a014d73d..5215eefd 100644 --- a/src/body/movenet.ts +++ b/src/body/movenet.ts @@ -45,7 +45,7 @@ async function parseSinglePose(res, config, image, inputBox) { for (let id = 0; id < kpt.length; id++) { score = kpt[id][2]; if (score > config.body.minConfidence) { - const positionRaw: [number, number] = [ + const positionRaw: Point = [ (inputBox[3] - inputBox[1]) * kpt[id][1] + inputBox[1], (inputBox[2] - inputBox[0]) * kpt[id][0] + inputBox[0], ]; @@ -93,7 +93,7 @@ async function parseMultiPose(res, config, image, inputBox) { for (let i = 0; i < 17; i++) { const partScore = Math.round(100 * kpt[3 * i + 2]) / 100; if (partScore > config.body.minConfidence) { - const positionRaw: [number, number] = [ + const positionRaw: Point = [ (inputBox[3] - inputBox[1]) * kpt[3 * i + 1] + inputBox[1], (inputBox[2] - inputBox[0]) * kpt[3 * i + 0] + inputBox[0], ]; diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index 6ecfb707..d4214cb7 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -9,7 +9,7 @@ import { log, join } from '../util/util'; import { scale } from '../util/box'; import * as tf from '../../dist/tfjs.esm.js'; -import type { HandResult, Box } from '../result'; +import type { HandResult, Box, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; @@ -25,7 +25,7 @@ const inputSize = [[0, 0], [0, 0]]; const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip']; let skipped = 0; -let outputSize: [number, number] = [0, 0]; +let outputSize: Point = [0, 0]; type HandDetectResult = { id: number, @@ -115,7 +115,7 @@ async function detectHands(input: Tensor, config: Config): Promise (config.hand.minConfidence || 0)) { hand.fingerScore = score; t.reshaped = tf.reshape(t.keypoints, [-1, 3]); - const rawCoords = await t.reshaped.array() as number[]; - hand.keypoints = (rawCoords as number[]).map((coord) => [ + const rawCoords = await t.reshaped.array() as Point[]; + hand.keypoints = (rawCoords as Point[]).map((coord) => [ (h.box[2] * coord[0] / inputSize[1][0]) + h.box[0], (h.box[3] * coord[1] / inputSize[1][1]) + h.box[1], - (h.box[2] + h.box[3]) / 2 / inputSize[1][0] * coord[2], + (h.box[2] + h.box[3]) / 2 / inputSize[1][0] * (coord[2] || 0), ]); const updatedBox = scale(hand.keypoints, boxScaleFact, outputSize); // replace detected box with box calculated around keypoints h.box = updatedBox.box; diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index 7fba0a31..4dbb395a 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -77,6 +77,7 @@ export async function check(instance, force = false) { tf.ENV.set('WEBGL_CPU_FORWARD', true); tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', false); tf.ENV.set('WEBGL_USE_SHAPES_UNIFORMS', true); + tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 128); // if (!instance.config.object.enabled) tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // safe to use 16bit precision if (typeof instance.config['deallocate'] !== 'undefined' && instance.config['deallocate']) { // hidden param log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);