From 334bb7061fa9c31bd52c60f28bf39b277efd7391 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 12 Aug 2021 09:31:16 -0400 Subject: [PATCH] switch to async data reads --- CHANGELOG.md | 7 +++-- TODO.md | 41 ++++++++++++++++++++++-------- demo/nodejs/node.js | 2 +- src/age/age.ts | 2 +- src/efficientpose/efficientpose.ts | 6 ++--- src/emotion/emotion.ts | 2 +- src/faceres/faceres.ts | 8 +++--- src/gender/gender.ts | 4 +-- src/handpose/handdetector.ts | 4 +-- src/human.ts | 2 +- src/object/centernet.ts | 2 +- src/object/nanodet.ts | 2 +- 12 files changed, 52 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c582f24..c557d467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **2.1.2** +Version: **2.1.3** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,7 +9,10 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/08/09 mandic00@live.com +### **2.1.3** 2021/08/12 mandic00@live.com + + +### **origin/main** 2021/08/11 mandic00@live.com - minor update - replace movenet with lightning-v4 diff --git a/TODO.md b/TODO.md index bb903c2b..92e16c2f 100644 --- a/TODO.md +++ b/TODO.md @@ -21,31 +21,50 @@ WebGL shader optimizations for faster load and initial detection - Optical Flow: - TFLite Models: +
+ ## Known Issues -### Object Detection - -Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS -*Target: `Human` v2.2 with `TFJS` v3.9* - -- CenterNet with WASM: -- NanoDet with WASM: +
### Face Detection Enhanced rotation correction for face detection is not working in NodeJS due to missing kernel op in TFJS Feature is automatically disabled in NodeJS without user impact -*Target: `Human` v2.2 with `TFJS` v3.9* -- BlazeFace rotation correction in NodeJS: +- Backend NodeJS missing kernel op `FlipLeftRight` + + *Target: `Human` v2.2 with `TFJS` v3.9* +- Backend NodeJS missing kernel op `RotateWithOffset` + + *Target: N/A* + +
### Hand Detection Enhanced rotation correction for hand detection is not working in NodeJS due to missing kernel op in TFJS Feature is automatically disabled in NodeJS without user impact -*Target: `Human` v2.2 with `TFJS` v3.9* -- HandPose rotation correction in NodeJS: +- Backend NodeJS missing kernel op `FlipLeftRight` + + *Target: `Human` v2.2 with `TFJS` v3.9* +- Backend NodeJS missing kernel op `RotateWithOffset` + + *Target: N/A* Hand detection using WASM backend has reduced precision due to math rounding errors in backend *Target: N/A* + +
+ +### Object Detection + +Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS + +- Backend WASM missing kernel op `Mod` + + *Target: `Human` v2.2 with `TFJS` v3.9* +- Backend WASM missing kernel op `SparseToDense` + + *Target: `Human` v2.2 with `TFJS` v3.9* diff --git a/demo/nodejs/node.js b/demo/nodejs/node.js index 3a5d5002..3cbb85f1 100644 --- a/demo/nodejs/node.js +++ b/demo/nodejs/node.js @@ -121,7 +121,7 @@ async function detect(input) { if (result && result.hand && result.hand.length > 0) { for (let i = 0; i < result.hand.length; i++) { const hand = result.hand[i]; - log.data(` Hand: #${i} score:${hand.score}`); + log.data(` Hand: #${i} score:${hand.score} keypoints:${hand.keypoints?.length}`); } } else { log.data(' Hand: N/A'); diff --git a/src/age/age.ts b/src/age/age.ts index e4f6b1e0..3f0e06f6 100644 --- a/src/age/age.ts +++ b/src/age/age.ts @@ -45,7 +45,7 @@ export async function predict(image: Tensor, config: Config | any) { tf.dispose(enhance); if (ageT) { - const data = ageT.dataSync(); + const data = await ageT.data(); obj.age = Math.trunc(10 * data[0]) / 10; } tf.dispose(ageT); diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index 956fe38e..dfe298bf 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -39,12 +39,12 @@ function max2d(inputs, minScore) { // combine all data const reshaped = tf.reshape(inputs, [height * width]); // get highest score - const newScore = tf.max(reshaped, 0).dataSync()[0]; + const newScore = tf.max(reshaped, 0).dataSync()[0]; // inside tf.tidy if (newScore > minScore) { // skip coordinate calculation is score is too low const coords = tf.argMax(reshaped, 0); - const x = mod(coords, width).dataSync()[0]; - const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; + const x = mod(coords, width).dataSync()[0]; // inside tf.tidy + const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy return [x, y, newScore]; } return [0, 0, newScore]; diff --git a/src/emotion/emotion.ts b/src/emotion/emotion.ts index 7764a31a..683fcd58 100644 --- a/src/emotion/emotion.ts +++ b/src/emotion/emotion.ts @@ -53,7 +53,7 @@ export async function predict(image: Tensor, config: Config, idx, count) { const obj: Array<{ score: number, emotion: string }> = []; if (config.face.emotion.enabled) { const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation - const data = emotionT.dataSync(); + const data = await emotionT.data(); tf.dispose(emotionT); for (let i = 0; i < data.length; i++) { if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index e55d9adb..dcf98d2c 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -134,21 +134,21 @@ export async function predict(image: Tensor, config: Config, idx, count) { if (resT) { tf.tidy(() => { - const gender = resT.find((t) => t.shape[1] === 1).dataSync(); + const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; if (confidence > config.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.genderScore = Math.min(0.99, confidence); } - const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; - const all = resT.find((t) => t.shape[1] === 100).dataSync(); + const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy + const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; const desc = resT.find((t) => t.shape[1] === 1024); // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8 // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor - obj.descriptor = [...desc.dataSync()]; + obj.descriptor = [...desc.dataSync()]; // inside tf.tidy }); resT.forEach((t) => tf.dispose(t)); } diff --git a/src/gender/gender.ts b/src/gender/gender.ts index d623f324..be88a409 100644 --- a/src/gender/gender.ts +++ b/src/gender/gender.ts @@ -63,7 +63,7 @@ export async function predict(image: Tensor, config: Config | any) { if (genderT) { if (!Array.isArray(genderT)) { - const data = genderT.dataSync(); + const data = await genderT.data(); if (alternative) { // returns two values 0..1, bigger one is prediction if (data[0] > config.face.gender.minConfidence || data[1] > config.face.gender.minConfidence) { @@ -80,7 +80,7 @@ export async function predict(image: Tensor, config: Config | any) { } tf.dispose(genderT); } else { - const gender = genderT[0].dataSync(); + const gender = await genderT[0].data(); const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; if (confidence > config.face.gender.minConfidence) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index e0ce14d4..3a574c6c 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -45,7 +45,7 @@ export class HandDetector { const predictions = tf.squeeze(batched); tf.dispose(batched); const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])))); - const scores = scoresT.dataSync(); + const scores = await scoresT.data(); const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); const boxes = this.normalizeBoxes(rawBoxes); tf.dispose(rawBoxes); @@ -78,7 +78,7 @@ export class HandDetector { const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = []; if (!predictions || predictions.length === 0) return hands; for (const prediction of predictions) { - const boxes = prediction.box.dataSync(); + const boxes = await prediction.box.data(); const startPoint = boxes.slice(0, 2); const endPoint = boxes.slice(2, 4); const palmLandmarks = await prediction.palmLandmarks.array(); diff --git a/src/human.ts b/src/human.ts index 73ee4976..60b4e92c 100644 --- a/src/human.ts +++ b/src/human.ts @@ -367,7 +367,7 @@ export class Human { sumT.dispose(); */ // use js loop sum, faster than uploading tensor to gpu calculating and downloading back - const reducedData = reduced.dataSync(); // raw image rgb array + const reducedData = await reduced.data(); // raw image rgb array let sum = 0; for (let i = 0; i < reducedData.length / 3; i++) sum += reducedData[3 * i + 2]; // look only at green value of each pixel diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 4942d13a..99d3f69d 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -42,7 +42,7 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) { tf.dispose(boxesT); tf.dispose(scoresT); tf.dispose(classesT); - const nms = nmsT.dataSync(); + const nms = await nmsT.data(); tf.dispose(nmsT); let i = 0; for (const id of nms) { diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 11b3800c..732153a6 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -90,7 +90,7 @@ async function process(res, inputSize, outputShape, config) { let nmsIdx: Array = []; if (nmsBoxes && nmsBoxes.length > 0) { const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); - nmsIdx = nms.dataSync(); + nmsIdx = await nms.data(); tf.dispose(nms); }