From ded141a1615119f9af99d048f40e5f182f9252a5 Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Mon, 20 Sep 2021 21:59:49 -0400
Subject: [PATCH] support for dynamic backend switching

---
 src/human.ts        | 34 ++++++++++--------
 src/tfjs/backend.ts |  6 ++--
 src/warmup.ts       | 18 ++++++----
 test/browser.html   | 88 +++++++++++++++++++++++++--------------------
 test/test-main.js   |  6 ++--
 5 files changed, 85 insertions(+), 67 deletions(-)

diff --git a/src/human.ts b/src/human.ts
index 467b1aa7..17800e41 100644
--- a/src/human.ts
+++ b/src/human.ts
@@ -2,7 +2,7 @@
  * Human main module
  */
 
-import { log, now, mergeDeep, validate, wait } from './helpers';
+import { log, now, mergeDeep, validate } from './helpers';
 import { Config, defaults } from './config';
 import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
 import * as tf from '../dist/tfjs.esm.js';
@@ -252,7 +252,11 @@ export class Human {
   }
 
   /** Reset configuration to default values */
-  reset = () => this.config = JSON.parse(JSON.stringify(defaults));
+  reset = () => {
+    const currentBackend = this.config.backend; // save backend;
+    this.config = JSON.parse(JSON.stringify(defaults));
+    this.config.backend = currentBackend;
+  }
 
   /** Validate current configuration schema */
   validate = (userConfig?: Partial<Config>) => validate(defaults, userConfig || this.config);
@@ -314,12 +318,13 @@ export class Human {
   /** Explicit backend initialization
    *  - Normally done implicitly during initial load phase
    *  - Call to explictly register and initialize TFJS backend without any other operations
-   *  - Used in webworker environments where there can be multiple instances of Human and not all initialized
+   *  - Use when changing backend during runtime
    *
    * @return Promise<void>
    */
-  init() {
-    backend.check(this);
+  async init() {
+    await backend.check(this, true);
+    await this.tf.ready();
     env.set(this.env);
   }
 
@@ -396,7 +401,7 @@ export class Human {
   */
   async detect(input: Input, userConfig?: Partial<Config>): Promise<Result | Error> {
     // detection happens inside a promise
-    if (this.config.yield) await wait(1);
+    this.state = 'detect';
     return new Promise(async (resolve) => {
       this.state = 'config';
       let timeStamp;
@@ -421,8 +426,8 @@ export class Human {
       // load models if enabled
       await this.load();
 
-      if (this.config.yield) await wait(1);
       timeStamp = now();
+      this.state = 'image';
       let img = image.process(input, this.config);
       this.process = img;
       this.performance.image = Math.trunc(now() - timeStamp);
@@ -431,7 +436,7 @@ export class Human {
       // run segmentation prethis.processing
       if (this.config.segmentation.enabled && this.process && img.tensor && img.canvas) {
         this.analyze('Start Segmentation:');
-        this.state = 'run:segmentation';
+        this.state = 'detect:segmentation';
         timeStamp = now();
         await segmentation.predict(img);
         elapsedTime = Math.trunc(now() - timeStamp);
@@ -468,7 +473,7 @@ export class Human {
       let objectRes: ObjectResult[] | Promise<ObjectResult[]> | never[] = [];
 
       // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
-      this.state = 'run:face';
+      this.state = 'detect:face';
       if (this.config.async) {
         faceRes = this.config.face.enabled ? face.detectFace(this, img.tensor) : [];
         if (this.performance.face) delete this.performance.face;
@@ -481,7 +486,7 @@ export class Human {
 
       // run body: can be posenet, blazepose, efficientpose, movenet
       this.analyze('Start Body:');
-      this.state = 'run:body';
+      this.state = 'detect:body';
       if (this.config.async) {
         if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(img.tensor, this.config) : [];
         else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(img.tensor, this.config) : [];
@@ -501,7 +506,7 @@ export class Human {
 
       // run handpose
       this.analyze('Start Hand:');
-      this.state = 'run:hand';
+      this.state = 'detect:hand';
       if (this.config.async) {
         handRes = this.config.hand.enabled ? handpose.predict(img.tensor, this.config) : [];
         if (this.performance.hand) delete this.performance.hand;
@@ -515,7 +520,7 @@ export class Human {
 
       // run nanodet
       this.analyze('Start Object:');
-      this.state = 'run:object';
+      this.state = 'detect:object';
       if (this.config.async) {
         if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(img.tensor, this.config) : [];
         else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(img.tensor, this.config) : [];
@@ -530,12 +535,11 @@ export class Human {
       this.analyze('End Object:');
 
       // if async wait for results
-      this.state = 'run:await';
-      if (this.config.yield) await wait(1);
+      this.state = 'detect:await';
       if (this.config.async) [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
 
       // run gesture analysis last
-      this.state = 'run:gesture';
+      this.state = 'detect:gesture';
       let gestureRes: GestureResult[] = [];
       if (this.config.gesture.enabled) {
         timeStamp = now();
diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts
index bc86bda2..d818541e 100644
--- a/src/tfjs/backend.ts
+++ b/src/tfjs/backend.ts
@@ -3,10 +3,10 @@ import * as humangl from './humangl';
 import * as env from '../env';
 import * as tf from '../../dist/tfjs.esm.js';
 
-export async function check(instance) {
-  if (env.env.initial || (instance.config.backend && (instance.config.backend.length > 0) && (tf.getBackend() !== instance.config.backend))) {
+export async function check(instance, force = false) {
+  instance.state = 'backend';
+  if (force || env.env.initial || (instance.config.backend && (instance.config.backend.length > 0) && (tf.getBackend() !== instance.config.backend))) {
     const timeStamp = now();
-    instance.state = 'backend';
 
     if (instance.config.backend && instance.config.backend.length > 0) {
       // detect web worker
diff --git a/src/warmup.ts b/src/warmup.ts
index 7ff04918..60e094fa 100644
--- a/src/warmup.ts
+++ b/src/warmup.ts
@@ -12,6 +12,7 @@ async function warmupBitmap(instance) {
   let res;
   switch (instance.config.warmup) {
     case 'face': blob = await b64toBlob(sample.face); break;
+    case 'body':
     case 'full': blob = await b64toBlob(sample.body); break;
     default: blob = null;
   }
@@ -98,14 +99,17 @@ async function warmupNode(instance) {
 */
 export async function warmup(instance, userConfig?: Partial<Config>): Promise<Result | { error }> {
   const t0 = now();
+  instance.state = 'warmup';
   if (userConfig) instance.config = mergeDeep(instance.config, userConfig) as Config;
   if (!instance.config.warmup || instance.config.warmup === 'none') return { error: 'null' };
   let res;
-  if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
-  else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
-  else res = await warmupNode(instance);
-  const t1 = now();
-  if (instance.config.debug) log('Warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
-  instance.emit('warmup');
-  return res;
+  return new Promise(async (resolve) => {
+    if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
+    else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
+    else res = await warmupNode(instance);
+    const t1 = now();
+    if (instance.config.debug) log('Warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
+    instance.emit('warmup');
+    resolve(res);
+  });
 }
diff --git a/test/browser.html b/test/browser.html
index 415dd2aa..0136dbb9 100644
--- a/test/browser.html
+++ b/test/browser.html
@@ -14,28 +14,30 @@
       @font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
       html { font-family: 'Lato', 'Segoe UI'; font-size: 14px; font-variant: small-caps; }
       body { margin: 0; background: black; color: white; }
-      canvas { position: absolute; bottom: 10px; right: 10px; width: 256px; height: 256px; }
-      pre { line-height: 150%; }
-      .events { position: absolute; top: 10px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; }
-      .state { position: absolute; top: 60px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; }
+      .canvas { position: absolute; bottom: 10px; right: 10px; width: 256px; height: 256px; z-index: 99; }
+      .pre { line-height: 150%; }
+      .events { position: absolute; top: 10px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; z-index: 99; }
+      .state { position: absolute; top: 60px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; z-index: 99; }
     </style>
   </head>
   <body>
-    <pre id="log"></pre>
+    <pre id="log" class="pre"></pre>
     <div id="events" class="events"></div>
     <div id="state" class="state"></div>
+    <canvas id="canvas" class="canvas" width=256 height=256></canvas>
     <script type="module">
       import Human from '../dist/human.esm.js';
 
       const config = {
         async: true,
-        warmup: 'full',
+        warmup: 'none',
         debug: true,
         cacheSensitivity: 0,
         object: { enabled: true },
       }
 
       const backends = ['wasm', 'webgl', 'humangl'];
+      // const backends = ['wasm', 'wasm'];
       // const backends = ['humangl'];
 
       const start = performance.now();
@@ -50,9 +52,14 @@
         return line + '\n';
       }
 
+      let last = new Date();
       async function log(...msgs) {
-        document.getElementById('log').innerHTML += str(...msgs);
-        console.log(...msgs);
+        const dt = new Date();
+        const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
+        const elap = (dt - last).toString().padStart(5, '0');
+        document.getElementById('log').innerHTML += ts + ' +' + elap + 'ms' + '&nbsp' + str(...msgs);
+        console.log(ts, elap, ...msgs);
+        last = dt;
       }
 
       async function image(url) {
@@ -69,6 +76,13 @@
         await waiting;
       }
 
+      function draw(canvas = null) {
+        const c = document.getElementById('canvas');
+        const ctx = c.getContext('2d');
+        if (canvas) ctx.drawImage(canvas, 0, 0, c.width, c.height);
+        else ctx.clearRect(0, 0, c.width, c.height);
+      }
+
       async function events(event) {
         document.getElementById('events').innerText = `${Math.round(performance.now() - start)}ms Event: ${event}`;
       }
@@ -77,66 +91,62 @@
         log('human tests');
         let res;
         let human = new Human(config);
-        setInterval(() => { document.getElementById('state').innerText = `State: ${human.state}`; }, 10);
+        human.events.addEventListener('warmup', () => events('warmup'));
+        human.events.addEventListener('image', () => events('image'));
+        human.events.addEventListener('detect', () => events('detect'));
+        const timer = setInterval(() => { document.getElementById('state').innerText = `State: ${human.state}`; }, 10);
         log({ version: human.version });
         log({ env: human.env });
         log({ config: human.config });
+        log({ tfjs: human.tf.version.tfjs, backend: human.config.backend });
+        await human.load();
+        const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) }));
+        log({ models });
         for (const backend of backends) {
-          log('');
+          log();
           log('test start:', backend);
           human.config.backend = backend;
-          human = new Human(config);
-          human.events.addEventListener('warmup', () => events('warmup'));
-          human.events.addEventListener('image', () => events('image'));
-          human.events.addEventListener('detect', () => events('detect'));
-          await human.load();
-          human.env.offscreen = false;
-          human.env.initial = false;
           await human.init();
-          log({ tfjs: human.tf.version.tfjs, backend: human.tf.getBackend() });
-          const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) }));
-          log({ models: { models }});
-          log({ memory: human.tf.engine().state });
+          log({ initialized: human.tf.getBackend() });
+          log({ memory: human.tf.memory() });
           res = await human.validate();
           log({ validate: res });
-          res = await human.warmup();
-          log({ warmup: res });
+          res = await human.warmup({ warmup: 'face'});
+          draw(res.canvas);
+          log({ warmup: 'face' });
           let img = await image('../../samples/ai-body.jpg');
           const input = await human.image(img);
-          let node = document.body.appendChild(res.canvas);
-          await wait(100);
-          log({ input });
+          log({ input: input.tensor.shape });
+          draw(res.canvas);
           res = await human.detect(input.tensor);
-          log({ detect: res});
+          log({ detect: true });
           const interpolated = human.next();
-          log({ interpolated });
+          log({ interpolated: true });
           const persons = res.persons;
-          log({ persons: { persons } });
+          log({ persons: true });
           log({ summary: { persons: persons.length, face: res.face.length, body: res.body.length, hand: res.hand.length, object: res.object.length, gesture: res.gesture.length }});
           log({ performance: human.performance });
           human.tf.dispose(input.tensor);
-          document.body.removeChild(node);
-          await wait(100);
+          draw();
 
           img = await image('../../samples/ai-face.jpg');
-          human.reset();
-          human.config.backend = backend;
           for (const val of [0, 0.25, 0.5, 0.75, 10]) {
             human.performance = {};
             const t0 = performance.now();
             for (let i = 0; i < 10; i++) {
-              res = await human.detect(img, { cacheSensitivity: val, filter: { pixelate: 5 * i } });
-              node = document.body.appendChild(res.canvas);
+              res = await human.detect(img, { cacheSensitivity: val, filter: { pixelate: 5 * i }, object: { enabled: false } });
+              draw(res.canvas);
             }
             const t1 = performance.now();
             log({ benchmark: { time: Math.round((t1 - t0) / 10), cacheSensitivity: val }, performance: human.performance });
-            await wait(100);
+            await wait(10);
           }
-          document.body.removeChild(node);
+          draw();
 
-          log({ memory: human.tf.engine().state });
+          log({ memory: human.tf.memory() });
         }
-        log('');
+        clearInterval(timer);
+        log();
         log('tests complete');
       }
 
diff --git a/test/test-main.js b/test/test-main.js
index e27b3c4f..e6645e91 100644
--- a/test/test-main.js
+++ b/test/test-main.js
@@ -233,8 +233,8 @@ async function test(Human, inputConfig) {
   res1 = Math.round(100 * human.similarity(desc1, desc2));
   res2 = Math.round(100 * human.similarity(desc1, desc3));
   res3 = Math.round(100 * human.similarity(desc2, desc3));
-  if (res1 !== 51 || res2 !== 49 || res3 !== 53) log('error', 'failed: face match ', res1, res2, res3);
-  else log('state', 'passed: face match');
+  if (res1 !== 51 || res2 !== 49 || res3 !== 53) log('error', 'failed: face similarity ', res1, res2, res3);
+  else log('state', 'passed: face similarity');
 
   // test face matching
   log('info', 'test face matching');
@@ -247,7 +247,7 @@ async function test(Human, inputConfig) {
   res1 = human.match(desc1, db);
   res2 = human.match(desc2, db);
   res3 = human.match(desc3, db);
-  if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1);
+  if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1, res2, res3);
   else log('state', 'passed: face match', { first: { name: res1.name, similarity: res1.similarity } }, { second: { name: res2.name, similarity: res2.similarity } }, { third: { name: res3.name, similarity: res3.similarity } });
 
   // test object detection