complete async work

demo/index.js

@@ -41,9 +41,9 @@ let userConfig = {
     flip: false,
   },
   face: { enabled: true,
-    detector: { return: false },
+    detector: { return: false, rotation: true },
     mesh: { enabled: true },
-    iris: { enabled: false },
+    iris: { enabled: true },
     description: { enabled: false },
     emotion: { enabled: false },
   },
@@ -441,9 +441,9 @@ function webWorker(input, image, canvas, timestamp) {
 // main processing function when input is webcam, can use direct invocation or web worker
 function runHumanDetect(input, canvas, timestamp) {
   // if live video
-  const videoLive = (input.readyState > 2) && (!input.paused);
+  const videoLive = input.readyState > 2;
-  const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live');
+  const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
-  const live = videoLive || cameraLive;
+  const live = (videoLive || cameraLive) && (!input.paused);
   if (!live) {
     // stop ui refresh
     // if (ui.drawThread) cancelAnimationFrame(ui.drawThread);

package.json

@@ -83,7 +83,7 @@
     "node-fetch": "^2.6.1",
     "rimraf": "^3.0.2",
     "seedrandom": "^3.0.5",
-    "simple-git": "^2.43.0",
+    "simple-git": "^2.44.0",
     "tslib": "^2.3.1",
     "typedoc": "0.21.5",
     "typescript": "4.3.5"

src/efficientpose/efficientpose.ts

@@ -34,14 +34,10 @@ export async function load(config: Config): Promise<GraphModel> {
 function max2d(inputs, minScore) {
   const [width, height] = inputs.shape;
   return tf.tidy(() => {
-    // modulus op implemented in tf
+    const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); // modulus op implemented in tf
-    const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32')));
+    const reshaped = tf.reshape(inputs, [height * width]); // combine all data
-    // combine all data
+    const newScore = tf.max(reshaped, 0).dataSync()[0]; // get highest score // inside tf.tidy
-    const reshaped = tf.reshape(inputs, [height * width]);
+    if (newScore > minScore) { // skip coordinate calculation is score is too low
-    // get highest score
-    const newScore = tf.max(reshaped, 0).dataSync()[0]; // inside tf.tidy
-    if (newScore > minScore) {
-      // skip coordinate calculation is score is too low
       const coords = tf.argMax(reshaped, 0);
       const x = mod(coords, width).dataSync()[0]; // inside tf.tidy
       const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy

src/faceres/faceres.ts

@@ -133,23 +133,23 @@ export async function predict(image: Tensor, config: Config, idx, count) {
     tf.dispose(enhanced);
 
     if (resT) {
-      tf.tidy(() => {
+      const gender = await resT.find((t) => t.shape[1] === 1).data();
-        const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy
+      const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
-        const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
+      if (confidence > config.face.description.minConfidence) {
-        if (confidence > config.face.description.minConfidence) {
+        obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
-          obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
+        obj.genderScore = Math.min(0.99, confidence);
-          obj.genderScore = Math.min(0.99, confidence);
+      }
-        }
+      const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1);
-        const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy
+      const age = (await argmax.data())[0];
-        const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy
+      const all = await resT.find((t) => t.shape[1] === 100).data(); // inside tf.tidy
-        obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
+      obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
 
-        const desc = resT.find((t) => t.shape[1] === 1024);
+      const desc = resT.find((t) => t.shape[1] === 1024);
-        // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
+      // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
-        // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
+      // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
 
-        obj.descriptor = [...desc.dataSync()]; // inside tf.tidy
+      const descriptor = await desc.data();
-      });
+      obj.descriptor = [...descriptor];
       resT.forEach((t) => tf.dispose(t));
     }
 

src/gender/gender.ts

@@ -87,10 +87,10 @@ export async function predict(image: Tensor, config: Config | any) {
           obj.confidence = Math.min(0.99, confidence);
         }
         /*
-        let age = genderT[1].argMax(1).dataSync()[0];
+        let age = (await genderT[1].argMax(1).data())[0];
-        const all = genderT[1].dataSync();
+        const all = await genderT[1].data();
         age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
-        const descriptor = genderT[1].dataSync();
+        const descriptor = await genderT[1].data();
         */
         genderT.forEach((t) => tf.dispose(t));
       }

src/handpose/handpipeline.ts

@@ -118,7 +118,7 @@ export class HandPipeline {
         tf.dispose(rotatedImage);
         const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>;
         tf.dispose(handImage);
-        const confidence = confidenceT.dataSync()[0];
+        const confidence = (await confidenceT.data())[0];
         tf.dispose(confidenceT);
         if (confidence >= config.hand.minConfidence) {
           const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);

src/human.ts

@@ -302,8 +302,23 @@ export class Human {
         if (typeof window === 'undefined' && typeof WorkerGlobalScope !== 'undefined' && this.config.debug) log('running inside web worker');
 
         // force browser vs node backend
-        if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl';
+        if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') {
-        if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) this.config.backend = 'tensorflow';
+          if (this.config.debug) log('override: backend set to tensorflow while running in browser');
+          this.config.backend = 'humangl';
+        }
+        if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) {
+          if (this.config.debug) log('override: backend set to webgl while running in nodejs');
+          this.config.backend = 'tensorflow';
+        }
+
+        const available = Object.keys(this.tf.engine().registryFactory);
+        if (this.config.debug) log('available backends:', available);
+
+        if (!available.includes(this.config.backend)) {
+          log(`error: backend ${this.config.backend} not found in registry`);
+          this.config.backend = this.tf.ENV.flags.IS_NODE ? 'tensorflow' : 'humangl';
+          log(`override: using backend ${this.config.backend} instead`);
+        }
 
         if (this.config.debug) log('setting backend:', this.config.backend);
 
@@ -363,7 +378,7 @@ export class Human {
     // use tensor sum
     /*
     const sumT = this.tf.sum(reduced);
-    const sum = sumT.dataSync()[0] as number;
+    const sum = await sumT.data()[0] as number;
     sumT.dispose();
     */
     // use js loop sum, faster than uploading tensor to gpu calculating and downloading back