experimental node-wasm support

CHANGELOG.md

@@ -1,6 +1,6 @@
 # @vladmandic/human  
 
-Version: **1.5.0**  
+Version: **1.5.1**  
 Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**  
 
 Author: **Vladimir Mandic <mandic00@live.com>**  
@@ -9,8 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/04/13 mandic00@live.com
+### **1.5.1** 2021/04/13 mandic00@live.com
 
+- fix for safari imagebitmap
+- refactored human.config and human.draw
 
 ### **1.4.3** 2021/04/12 mandic00@live.com
 

TODO.md

@@ -2,19 +2,15 @@
 
 ## Big Ticket Items
 
-- Strong(er) typing
-- Automated testing framework
-- TypeDoc comments
+- Improve automated testing framework
 
 ## Explore Models
 
-- EfficientPose  
-  <https://github.com/daniegr/EfficientPose>  
-  <https://github.com/PINTO0309/PINTO_model_zoo/tree/main/084_EfficientPose>
 - InsightFace  
-  RetinaFace detetor and ArcFace recognition
+  RetinaFace detector and ArcFace recognition
   <https://github.com/deepinsight/insightface>  
 
 ## Issues
 
 - box sizing on mobile
+- canvas.js for wasm on node

package.json

@@ -24,7 +24,7 @@
     "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js",
     "dev": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/serve.js",
     "build": "rimraf dist/* typedoc/* types/* && node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/build.js",
-    "lint": "eslint src server demo",
+    "lint": "eslint src server demo test",
     "test": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation test/test-node.js"
   },
   "keywords": [

server/build.js

@@ -99,6 +99,22 @@ const targets = {
       external: ['@tensorflow'],
     },
   },
+  nodeWASM: {
+    tfjs: {
+      platform: 'node',
+      format: 'cjs',
+      entryPoints: ['src/tfjs/tf-node-wasm.ts'],
+      outfile: 'dist/tfjs.esm.js',
+      external: ['@tensorflow'],
+    },
+    node: {
+      platform: 'node',
+      format: 'cjs',
+      entryPoints: ['src/human.ts'],
+      outfile: 'dist/human.node-wasm.js',
+      external: ['@tensorflow'],
+    },
+  },
 
   browserNoBundle: {
     tfjs: {

src/human.ts

@@ -329,13 +329,14 @@ export class Human {
       if (this.config.backend && this.config.backend.length > 0) {
         // force browser vs node backend
         if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl';
-        if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'wasm')) this.config.backend = 'tensorflow';
+        if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) this.config.backend = 'tensorflow';
 
         if (this.config.debug) log('setting backend:', this.config.backend);
 
         if (this.config.backend === 'wasm') {
           if (this.config.debug) log('wasm path:', this.config.wasmPath);
-          this.tf.setWasmPaths(this.config.wasmPath);
+          if (typeof this.tf?.setWasmPaths !== 'undefined') this.tf.setWasmPaths(this.config.wasmPath);
+          else throw new Error('Human: WASM backend is not loaded');
           const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
           const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
           if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`);
@@ -573,8 +574,16 @@ export class Human {
 
   /** @hidden */
   #warmupNode = async () => {
+    // @ts-ignore
+    if (typeof tf.node === 'undefined') {
+      if (this.config.debug) log('Warmup tfjs-node not loaded');
+      return null;
+    }
     const atob = (str) => Buffer.from(str, 'base64');
-    const img = this.config.warmup === 'face' ? atob(sample.face) : atob(sample.body);
+    let img;
+    if (this.config.warmup === 'face') img = atob(sample.face);
+    if (this.config.warmup === 'body' || this.config.warmup === 'full') img = atob(sample.body);
+    if (!img) return null;
     // @ts-ignore // tf.node is only defined when compiling for nodejs
     const data = tf.node?.decodeJpeg(img);
     const expanded = data.expandDims(0);
@@ -592,6 +601,7 @@ export class Human {
   async warmup(userConfig: Config | Object = {}): Promise<Result | { error }> {
     const t0 = now();
     if (userConfig) this.config = mergeDeep(this.config, userConfig);
+    if (!this.config.warmup || this.config.warmup === 'none') return { error: 'null' };
     const save = this.config.videoOptimized;
     this.config.videoOptimized = false;
     let res;

src/tfjs/tf-node-wasm.ts

@@ -0,0 +1,2 @@
+export * from '@tensorflow/tfjs';
+export * from '@tensorflow/tfjs-backend-wasm';

test/test-node.js

@@ -50,36 +50,51 @@ async function testInstance(human) {
     log.error('failed: load models');
   }
 
-  const warmup = await human.warmup();
+  let warmup;
+  try {
+    warmup = await human.warmup();
+  } catch (err) {
+    log.error('error warmup');
+  }
   if (warmup) {
     log.state('passed: warmup:', config.warmup);
-    log.data(' result: face:', warmup.face.length, 'body:', warmup.body.length, 'hand:', warmup.hand.length, 'gesture:', warmup.gesture.length, 'object:', warmup.object.length);
-    log.data(' result: performance:', 'load:', warmup.performance.load, 'total:', warmup.performance.total);
+    log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length);
+    log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total);
   } else {
     log.error('failed: warmup');
   }
   const random = tf.randomNormal([1, 1024, 1024, 3]);
-  const detect = await human.detect(random);
+  let detect;
+  try {
+    detect = await human.detect(random);
+  } catch (err) {
+    log.error('error: detect', err);
+  }
   tf.dispose(random);
   if (detect) {
     log.state('passed: detect:', 'random');
-    log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length);
-    log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total);
+    log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length);
+    log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
   } else {
     log.error('failed: detect');
   }
 }
 
 async function test() {
-  log.info('testing instance#1');
-  config.warmup = 'face';
+  log.info('testing instance#1 - none');
+  config.warmup = 'none';
   const human1 = new Human(config);
   await testInstance(human1);
 
-  log.info('testing instance#2');
-  config.warmup = 'body';
+  log.info('testing instance#2 - face');
+  config.warmup = 'face';
   const human2 = new Human(config);
   await testInstance(human2);
+
+  log.info('testing instance#3 - body');
+  config.warmup = 'body';
+  const human3 = new Human(config);
+  await testInstance(human3);
 }
 
 test();

test/test-wasm.js

@@ -0,0 +1,99 @@
+const log = require('@vladmandic/pilogger');
+const tf = require('@tensorflow/tfjs');
+const Human = require('../dist/human.node-wasm.js').default;
+
+const config = {
+  backend: 'wasm',
+  wasmPath: 'assets/',
+  debug: false,
+  videoOptimized: false,
+  async: false,
+  modelBasePath: 'http://localhost:10030/models',
+  filter: {
+    enabled: true,
+  },
+  face: {
+    enabled: true,
+    detector: { enabled: true, rotation: false },
+    mesh: { enabled: true },
+    iris: { enabled: true },
+    description: { enabled: true },
+    emotion: { enabled: true },
+  },
+  hand: {
+    enabled: true,
+  },
+  // body: { modelPath: 'efficientpose.json', enabled: true },
+  // body: { modelPath: 'blazepose.json', enabled: true },
+  body: { modelPath: 'posenet.json', enabled: true },
+  object: { enabled: false }, // Error: Kernel 'SparseToDense' not registered for backend 'wasm'
+};
+
+async function testInstance(human) {
+  if (human) log.state('passed: create human');
+  else log.error('failed: create human');
+
+  // if (!human.tf) human.tf = tf;
+  log.info('human version:', human.version);
+  log.info('tfjs version:', human.tf.version_core);
+  log.info('platform:', human.sysinfo.platform);
+  log.info('agent:', human.sysinfo.agent);
+
+  await human.load();
+  if (human.models) {
+    log.state('passed: load models');
+    const keys = Object.keys(human.models);
+    const loaded = keys.filter((model) => human.models[model]);
+    log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length);
+  } else {
+    log.error('failed: load models');
+  }
+
+  let warmup;
+  try {
+    warmup = await human.warmup();
+  } catch (err) {
+    log.error('error warmup');
+  }
+  if (warmup) {
+    log.state('passed: warmup:', config.warmup);
+    log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length);
+    log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total);
+  } else {
+    log.error('failed: warmup');
+  }
+  const random = tf.randomNormal([1, 1024, 1024, 3]);
+  let detect;
+  try {
+    detect = await human.detect(random);
+  } catch (err) {
+    log.error('error: detect', err);
+  }
+  tf.dispose(random);
+  if (detect) {
+    log.state('passed: detect:', 'random');
+    log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length);
+    log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
+  } else {
+    log.error('failed: detect');
+  }
+}
+
+async function test() {
+  log.info('testing instance#1 - none');
+  config.warmup = 'none';
+  const human1 = new Human(config);
+  await testInstance(human1);
+
+  log.info('testing instance#2 - face');
+  config.warmup = 'face';
+  const human2 = new Human(config);
+  await testInstance(human2);
+
+  log.info('testing instance#3 - body');
+  config.warmup = 'body';
+  const human3 = new Human(config);
+  await testInstance(human3);
+}
+
+test();