switch to async data reads

CHANGELOG.md

@@ -1,6 +1,6 @@
 # @vladmandic/human  
 
-Version: **2.1.2**  
+Version: **2.1.3**  
 Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
 
 Author: **Vladimir Mandic <mandic00@live.com>**  
@@ -9,7 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/08/09 mandic00@live.com
+### **2.1.3** 2021/08/12 mandic00@live.com
+
+
+### **origin/main** 2021/08/11 mandic00@live.com
 
 - minor update
 - replace movenet with lightning-v4

TODO.md

@@ -21,31 +21,50 @@ WebGL shader optimizations for faster load and initial detection
 - Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
 - TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
 
+<br>
+
 ## Known Issues
 
-### Object Detection
+<br>
-
-Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS  
-*Target: `Human` v2.2 with `TFJS` v3.9*
-
-- CenterNet with WASM: <https://github.com/tensorflow/tfjs/issues/5110>
-- NanoDet with WASM: <https://github.com/tensorflow/tfjs/issues/4824>
 
 ### Face Detection
 
 Enhanced rotation correction for face detection is not working in NodeJS due to missing kernel op in TFJS  
 Feature is automatically disabled in NodeJS without user impact  
-*Target: `Human` v2.2 with `TFJS` v3.9*
 
-- BlazeFace rotation correction in NodeJS: <https://github.com/tensorflow/tfjs/issues/4066>
+- Backend NodeJS missing kernel op `FlipLeftRight`  
+  <https://github.com/tensorflow/tfjs/issues/4066>  
+  *Target: `Human` v2.2 with `TFJS` v3.9*
+- Backend NodeJS missing kernel op `RotateWithOffset`  
+  <https://github.com/tensorflow/tfjs/issues/5473>  
+  *Target: N/A*
+
+<br>
 
 ### Hand Detection
 
 Enhanced rotation correction for hand detection is not working in NodeJS due to missing kernel op in TFJS  
 Feature is automatically disabled in NodeJS without user impact  
-*Target: `Human` v2.2 with `TFJS` v3.9*
 
-- HandPose rotation correction in NodeJS: <https://github.com/tensorflow/tfjs/issues/4066>
+- Backend NodeJS missing kernel op `FlipLeftRight`  
+  <https://github.com/tensorflow/tfjs/issues/4066>  
+  *Target: `Human` v2.2 with `TFJS` v3.9*
+- Backend NodeJS missing kernel op `RotateWithOffset`  
+  <https://github.com/tensorflow/tfjs/issues/5473>  
+  *Target: N/A*
 
 Hand detection using WASM backend has reduced precision due to math rounding errors in backend  
 *Target: N/A*
+
+<br>
+
+### Object Detection
+
+Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS  
+
+- Backend WASM missing kernel op `Mod`  
+  <https://github.com/tensorflow/tfjs/issues/5110>  
+  *Target: `Human` v2.2 with `TFJS` v3.9*
+- Backend WASM missing kernel op `SparseToDense`  
+  <https://github.com/tensorflow/tfjs/issues/4824>  
+  *Target: `Human` v2.2 with `TFJS` v3.9*

demo/nodejs/node.js

@@ -121,7 +121,7 @@ async function detect(input) {
   if (result && result.hand && result.hand.length > 0) {
     for (let i = 0; i < result.hand.length; i++) {
       const hand = result.hand[i];
-      log.data(`  Hand: #${i} score:${hand.score}`);
+      log.data(`  Hand: #${i} score:${hand.score} keypoints:${hand.keypoints?.length}`);
     }
   } else {
     log.data('  Hand: N/A');

src/age/age.ts

@@ -45,7 +45,7 @@ export async function predict(image: Tensor, config: Config | any) {
     tf.dispose(enhance);
 
     if (ageT) {
-      const data = ageT.dataSync();
+      const data = await ageT.data();
       obj.age = Math.trunc(10 * data[0]) / 10;
     }
     tf.dispose(ageT);

src/efficientpose/efficientpose.ts

@@ -39,12 +39,12 @@ function max2d(inputs, minScore) {
     // combine all data
     const reshaped = tf.reshape(inputs, [height * width]);
     // get highest score
-    const newScore = tf.max(reshaped, 0).dataSync()[0];
+    const newScore = tf.max(reshaped, 0).dataSync()[0]; // inside tf.tidy
     if (newScore > minScore) {
       // skip coordinate calculation is score is too low
       const coords = tf.argMax(reshaped, 0);
-      const x = mod(coords, width).dataSync()[0];
+      const x = mod(coords, width).dataSync()[0]; // inside tf.tidy
-      const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0];
+      const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy
       return [x, y, newScore];
     }
     return [0, 0, newScore];

src/emotion/emotion.ts

@@ -53,7 +53,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
     const obj: Array<{ score: number, emotion: string }> = [];
     if (config.face.emotion.enabled) {
       const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
-      const data = emotionT.dataSync();
+      const data = await emotionT.data();
       tf.dispose(emotionT);
       for (let i = 0; i < data.length; i++) {
         if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });

src/faceres/faceres.ts

@@ -134,21 +134,21 @@ export async function predict(image: Tensor, config: Config, idx, count) {
 
     if (resT) {
       tf.tidy(() => {
-        const gender = resT.find((t) => t.shape[1] === 1).dataSync();
+        const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy
         const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
         if (confidence > config.face.description.minConfidence) {
           obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
           obj.genderScore = Math.min(0.99, confidence);
         }
-        const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
+        const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy
-        const all = resT.find((t) => t.shape[1] === 100).dataSync();
+        const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy
         obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
 
         const desc = resT.find((t) => t.shape[1] === 1024);
         // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
         // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
 
-        obj.descriptor = [...desc.dataSync()];
+        obj.descriptor = [...desc.dataSync()]; // inside tf.tidy
       });
       resT.forEach((t) => tf.dispose(t));
     }

src/gender/gender.ts

@@ -63,7 +63,7 @@ export async function predict(image: Tensor, config: Config | any) {
 
     if (genderT) {
       if (!Array.isArray(genderT)) {
-        const data = genderT.dataSync();
+        const data = await genderT.data();
         if (alternative) {
           // returns two values 0..1, bigger one is prediction
           if (data[0] > config.face.gender.minConfidence || data[1] > config.face.gender.minConfidence) {
@@ -80,7 +80,7 @@ export async function predict(image: Tensor, config: Config | any) {
         }
         tf.dispose(genderT);
       } else {
-        const gender = genderT[0].dataSync();
+        const gender = await genderT[0].data();
         const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
         if (confidence > config.face.gender.minConfidence) {
           obj.gender = gender[0] <= 0.5 ? 'female' : 'male';

src/handpose/handdetector.ts

@@ -45,7 +45,7 @@ export class HandDetector {
     const predictions = tf.squeeze(batched);
     tf.dispose(batched);
     const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1]))));
-    const scores = scoresT.dataSync();
+    const scores = await scoresT.data();
     const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
     const boxes = this.normalizeBoxes(rawBoxes);
     tf.dispose(rawBoxes);
@@ -78,7 +78,7 @@ export class HandDetector {
     const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
     if (!predictions || predictions.length === 0) return hands;
     for (const prediction of predictions) {
-      const boxes = prediction.box.dataSync();
+      const boxes = await prediction.box.data();
       const startPoint = boxes.slice(0, 2);
       const endPoint = boxes.slice(2, 4);
       const palmLandmarks = await prediction.palmLandmarks.array();

src/human.ts

@@ -367,7 +367,7 @@ export class Human {
     sumT.dispose();
     */
     // use js loop sum, faster than uploading tensor to gpu calculating and downloading back
-    const reducedData = reduced.dataSync(); // raw image rgb array
+    const reducedData = await reduced.data(); // raw image rgb array
     let sum = 0;
     for (let i = 0; i < reducedData.length / 3; i++) sum += reducedData[3 * i + 2]; // look only at green value of each pixel
 

src/object/centernet.ts

@@ -42,7 +42,7 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) {
   tf.dispose(boxesT);
   tf.dispose(scoresT);
   tf.dispose(classesT);
-  const nms = nmsT.dataSync();
+  const nms = await nmsT.data();
   tf.dispose(nmsT);
   let i = 0;
   for (const id of nms) {

src/object/nanodet.ts

@@ -90,7 +90,7 @@ async function process(res, inputSize, outputShape, config) {
   let nmsIdx: Array<number> = [];
   if (nmsBoxes && nmsBoxes.length > 0) {
     const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
-    nmsIdx = nms.dataSync();
+    nmsIdx = await nms.data();
     tf.dispose(nms);
   }