update tfjs 3.4.0

.eslintrc.json

@@ -61,6 +61,7 @@
     "node/no-unpublished-import": "off",
     "node/no-unpublished-require": "off",
     "node/no-unsupported-features/es-syntax": "off",
+    "no-lonely-if": "off",
     "node/shebang": "off",
     "object-curly-newline": "off",
     "prefer-destructuring": "off",

CHANGELOG.md

@@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
+### **HEAD -> main** 2021/04/13 mandic00@live.com
+
+
 ### **1.5.1** 2021/04/13 mandic00@live.com
 
 - fix for safari imagebitmap

demo/helpers/webrtc.js

@@ -41,7 +41,6 @@ async function webRTC(server, streamName, elementName) {
   connection.ontrack = (event) => {
     stream.addTrack(event.track);
     const video = (typeof elementName === 'string') ? document.getElementById(elementName) : elementName;
-    // @ts-ignore
     if (video instanceof HTMLVideoElement) video.srcObject = stream;
     else log('element is not a video element:', elementName);
     // video.onloadeddata = async () => log('resolution:', video.videoWidth, video.videoHeight);

demo/index.js

@@ -198,7 +198,7 @@ async function setupCamera() {
   ui.busy = true;
   const viewportScale = Math.min(1, Math.round(100 * window.outerWidth / 700) / 100);
   // log('demo viewport scale:', viewportScale);
-  document.querySelector('meta[name=viewport]').setAttribute('content', `width=device-width, shrink-to-fit=no; initial-scale=${viewportScale}`);
+  document.querySelector('meta[name=viewport]').setAttribute('content', `width=device-width, shrink-to-fit=no, initial-scale=${viewportScale}`);
   const video = document.getElementById('video');
   const canvas = document.getElementById('canvas');
   const output = document.getElementById('log');
@@ -587,7 +587,7 @@ async function main() {
     log('overriding worker:', ui.useWorker);
   }
   if (params.has('backend')) {
-    userConfig.backend = JSON.parse(params.get('backend'));
+    userConfig.backend = params.get('backend');
     log('overriding backend:', userConfig.backend);
   }
   if (params.has('preload')) {

package.json

@@ -51,20 +51,21 @@
     "tensorflow"
   ],
   "devDependencies": {
-    "@tensorflow/tfjs": "^3.3.0",
+    "@tensorflow/tfjs": "^3.4.0",
-    "@tensorflow/tfjs-backend-cpu": "^3.3.0",
+    "@tensorflow/tfjs-backend-cpu": "^3.4.0",
-    "@tensorflow/tfjs-backend-wasm": "^3.3.0",
+    "@tensorflow/tfjs-backend-wasm": "^3.4.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.3.0",
+    "@tensorflow/tfjs-backend-webgl": "^3.4.0",
-    "@tensorflow/tfjs-converter": "^3.3.0",
+    "@tensorflow/tfjs-converter": "^3.4.0",
-    "@tensorflow/tfjs-core": "^3.3.0",
+    "@tensorflow/tfjs-core": "^3.4.0",
-    "@tensorflow/tfjs-data": "^3.3.0",
+    "@tensorflow/tfjs-data": "^3.4.0",
-    "@tensorflow/tfjs-layers": "^3.3.0",
+    "@tensorflow/tfjs-layers": "^3.4.0",
-    "@tensorflow/tfjs-node": "^3.3.0",
+    "@tensorflow/tfjs-node": "^3.4.0",
-    "@tensorflow/tfjs-node-gpu": "^3.3.0",
+    "@tensorflow/tfjs-node-gpu": "^3.4.0",
     "@types/node": "^14.14.37",
     "@typescript-eslint/eslint-plugin": "^4.22.0",
     "@typescript-eslint/parser": "^4.22.0",
     "@vladmandic/pilogger": "^0.2.16",
+    "canvas": "^2.7.0",
     "chokidar": "^3.5.1",
     "dayjs": "^1.10.4",
     "esbuild": "^0.11.10",
@@ -75,9 +76,10 @@
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "hint": "^6.1.3",
+    "node-fetch": "^2.6.1",
     "rimraf": "^3.0.2",
     "seedrandom": "^3.0.5",
-    "simple-git": "^2.37.0",
+    "simple-git": "^2.38.0",
     "tslib": "^2.2.0",
     "typedoc": "^0.20.35",
     "typescript": "^4.2.4"

src/human.ts

@@ -157,7 +157,7 @@ export class Human {
       faceres: null,
     };
     // export access to image processing
-    // @ts-ignore // typescript cannot infer type
+    // @ts-ignore eslint-typescript cannot correctly infer type in anonymous function
     this.image = (input: Input) => image.process(input, this.config);
     // export raw access to underlying models
     this.classes = {
@@ -266,9 +266,7 @@ export class Human {
         this.models.gender,
         this.models.emotion,
         this.models.embedding,
-        // @ts-ignore // typescript cannot infer type
         this.models.handpose,
-        // @ts-ignore // typescript cannot infer type
         this.models.posenet,
         this.models.blazepose,
         this.models.efficientpose,
@@ -280,8 +278,8 @@ export class Human {
         this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null),
         this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null),
         this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null),
-        this.models.handpose || (this.config.hand.enabled ? <Promise<handpose.HandPose>>handpose.load(this.config) : null),
+        this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
-        this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
+        this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? <any>posenet.load(this.config) : null),
         this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
         this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null),
         this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null),
@@ -340,7 +338,7 @@ export class Human {
           const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
           const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
           if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`);
-          if (!simd) log('warning: wasm simd support is not enabled');
+          if (this.config.debug && !simd) log('warning: wasm simd support is not enabled');
         }
 
         if (this.config.backend === 'humangl') backend.register();
@@ -574,23 +572,29 @@ export class Human {
 
   /** @hidden */
   #warmupNode = async () => {
-    // @ts-ignore
-    if (typeof tf.node === 'undefined') {
-      if (this.config.debug) log('Warmup tfjs-node not loaded');
-      return null;
-    }
     const atob = (str) => Buffer.from(str, 'base64');
     let img;
     if (this.config.warmup === 'face') img = atob(sample.face);
     if (this.config.warmup === 'body' || this.config.warmup === 'full') img = atob(sample.body);
     if (!img) return null;
-    // @ts-ignore // tf.node is only defined when compiling for nodejs
+    let res;
-    const data = tf.node?.decodeJpeg(img);
+    if (typeof tf['node'] !== 'undefined') {
-    const expanded = data.expandDims(0);
+      const data = tf['node'].decodeJpeg(img);
-    this.tf.dispose(data);
+      const expanded = data.expandDims(0);
-    // log('Input:', expanded);
+      this.tf.dispose(data);
-    const res = await this.detect(expanded, this.config);
+      // log('Input:', expanded);
-    this.tf.dispose(expanded);
+      res = await this.detect(expanded, this.config);
+      this.tf.dispose(expanded);
+    } else {
+      if (this.config.debug) log('Warmup tfjs-node not loaded');
+      /*
+      const input = await canvasJS.loadImage(img);
+      const canvas = canvasJS.createCanvas(input.width, input.height);
+      const ctx = canvas.getContext('2d');
+      ctx.drawImage(img, 0, 0, input.width, input.height);
+      res = await this.detect(input, this.config);
+      */
+    }
     return res;
   }
 

test/README.md

@@ -0,0 +1,47 @@
+# Test Results
+
+## Automatic Tests
+
+### NodeJS using TensorFlow library
+
+- Face rotation is disabled for `NodeJS` platform:  
+  `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'`  
+  <https://github.com/tensorflow/tfjs/issues/4606>
+- Image filters are disabled due to lack of Canvas and WeBGL access
+
+### NodeJS with GPU acceleation using CUDA
+
+- Face rotation is disabled for `NodeJS` platform:  
+  `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'`  
+  <https://github.com/tensorflow/tfjs/issues/4606>
+- Image filters are disabled due to lack of Canvas and WeBGL access
+
+### NodeJS using WASM
+
+- Requires dev http server  
+  See <https://github.com/tensorflow/tfjs/issues/4927>
+- Only supported input is Tensor due to missing image decoders
+- Warmup returns null and is marked as failed  
+  Missing native Image implementation in NodeJS
+- Fails on object detection:  
+  `Kernel 'SparseToDense' not registered for backend 'wasm'`  
+  <https://github.com/tensorflow/tfjs/issues/4824>
+
+<br>
+
+## Manual Tests
+
+### Browser using WebGL backend
+
+- Chrome/Edge: All Passing
+- Firefox: WebWorkers not supported due to missing support for OffscreenCanvas
+- Safari: Limited Testing
+
+### Browser using WASM backend
+
+- Chrome/Edge: All Passing
+- Firefox: WebWorkers not supported due to missing support for OffscreenCanvas
+- Safari: Limited Testing
+- Fails on object detection:  
+  `Kernel 'SparseToDense' not registered for backend 'wasm'`  
+  <https://github.com/tensorflow/tfjs/issues/4824>

test/test-main.js

@@ -0,0 +1,162 @@
+const process = require('process');
+const path = require('path');
+const log = require('@vladmandic/pilogger');
+const canvasJS = require('canvas');
+const fetch = require('node-fetch').default;
+
+let config;
+log.info('test:', path.basename(process.argv[1]));
+
+async function testHTTP() {
+  if (config.modelBasePath.startsWith('file:')) return true;
+  return new Promise((resolve) => {
+    fetch(config.modelBasePath)
+      .then((res) => {
+        if (res && res.ok) log.state('passed: model server:', config.modelBasePath);
+        else log.error('failed: model server:', config.modelBasePath);
+        resolve(res && res.ok);
+      })
+      .catch((err) => {
+        log.error('failed: model server:', err.message);
+        resolve(false);
+      });
+  });
+}
+
+async function getImage(human, input) {
+  let img;
+  try {
+    img = await canvasJS.loadImage(input);
+  } catch (err) {
+    log.error('failed: load image', input, err.message);
+    return img;
+  }
+  const canvas = canvasJS.createCanvas(img.width, img.height);
+  const ctx = canvas.getContext('2d');
+  ctx.drawImage(img, 0, 0, img.width, img.height);
+  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+  const res = human.tf.tidy(() => {
+    const tensor = human.tf.tensor(Array.from(imageData.data), [canvas.height, canvas.width, 4], 'int32'); // create rgba image tensor from flat array
+    const channels = human.tf.split(tensor, 4, 2); // split rgba to channels
+    const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb
+    const reshape = human.tf.reshape(rgb, [1, canvas.width, canvas.height, 3]); // move extra dim from the end of tensor and use it as batch number instead
+    return reshape;
+  });
+  if (res && res.shape[0] === 1 && res.shape[3] === 3) log.state('passed: load image:', input, res.shape);
+  else log.error('failed: load image:', input, res);
+  return res;
+}
+
+function printResults(detect) {
+  const person = (detect.face && detect.face[0]) ? { confidence: detect.face[0].confidence, age: detect.face[0].age, gender: detect.face[0].gender } : {};
+  const object = (detect.object && detect.object[0]) ? { score: detect.object[0].score, class: detect.object[0].label } : {};
+  const body = (detect.body && detect.body[0]) ? { score: detect.body[0].score, keypoints: detect.body[0].keypoints.length } : {};
+  if (detect.face) log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length, person, object, body);
+  if (detect.performance) log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
+}
+
+async function testInstance(human) {
+  if (human) log.state('passed: create human');
+  else log.error('failed: create human');
+
+  // if (!human.tf) human.tf = tf;
+  log.info('human version:', human.version);
+  log.info('platform:', human.sysinfo.platform, 'agent:', human.sysinfo.agent);
+  log.info('tfjs version:', human.tf.version.tfjs);
+
+  await human.load();
+  if (config.backend === human.tf.getBackend()) log.state('passed: set backend:', config.backend);
+  else log.error('failed: set backend:', config.backend);
+
+  if (human.models) {
+    log.state('passed: load models');
+    const keys = Object.keys(human.models);
+    const loaded = keys.filter((model) => human.models[model]);
+    log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length);
+    return true;
+  }
+  log.error('failed: load models');
+  return false;
+}
+
+async function testWarmup(human, title) {
+  let warmup;
+  try {
+    warmup = await human.warmup(config);
+  } catch (err) {
+    log.error('error warmup');
+  }
+  if (warmup) {
+    log.state('passed: warmup:', config.warmup, title);
+    printResults(warmup);
+    return true;
+  }
+  log.error('failed: warmup:', config.warmup, title);
+  return false;
+}
+
+async function testDetect(human, input, title) {
+  const image = input ? await getImage(human, input) : human.tf.randomNormal([1, 1024, 1024, 3]);
+  if (!image) {
+    log.error('failed: detect: input is null');
+    return false;
+  }
+  let detect;
+  try {
+    detect = await human.detect(image, config);
+  } catch (err) {
+    log.error('error: detect', err);
+  }
+  if (image instanceof human.tf.Tensor) human.tf.dispose(image);
+  if (detect) {
+    log.state('passed: detect:', input || 'random', title);
+    printResults(detect);
+    return true;
+  }
+  log.error('failed: detect', input || 'random', title);
+  return false;
+}
+
+async function test(Human, inputConfig) {
+  config = inputConfig;
+  const ok = await testHTTP();
+  if (!ok) {
+    log.warn('aborting test');
+    return;
+  }
+  const t0 = process.hrtime.bigint();
+  const human = new Human(config);
+  await testInstance(human);
+  config.warmup = 'none';
+  await testWarmup(human, 'default');
+  config.warmup = 'face';
+  await testWarmup(human, 'default');
+  config.warmup = 'body';
+  await testWarmup(human, 'default');
+
+  log.info('test body variants');
+  config.body = { modelPath: 'posenet.json', enabled: true };
+  await testDetect(human, 'assets/human-sample-body.jpg', 'posenet');
+  config.body = { modelPath: 'efficientpose.json', enabled: true };
+  await testDetect(human, 'assets/human-sample-body.jpg', 'efficientpose');
+  config.body = { modelPath: 'blazepose.json', enabled: true };
+  await testDetect(human, 'assets/human-sample-body.jpg', 'blazepose');
+
+  await testDetect(human, null, 'default');
+  log.info('test: first instance');
+  await testDetect(human, 'assets/sample-me.jpg', 'default');
+  log.info('test: second instance');
+  const second = new Human(config);
+  await testDetect(second, 'assets/sample-me.jpg', 'default');
+  log.info('test: concurrent');
+  await Promise.all([
+    testDetect(human, 'assets/human-sample-face.jpg', 'default'),
+    testDetect(second, 'assets/human-sample-face.jpg', 'default'),
+    testDetect(human, 'assets/human-sample-body.jpg', 'default'),
+    testDetect(second, 'assets/human-sample-body.jpg', 'default'),
+  ]);
+  const t1 = process.hrtime.bigint();
+  log.info('test complete:', Math.trunc(parseInt((t1 - t0).toString()) / 1000 / 1000), 'ms');
+}
+
+exports.test = test;

test/test-node-gpu.js

@@ -0,0 +1,26 @@
+const Human = require('../dist/human.node-gpu.js').default;
+const test = require('./test-main.js').test;
+
+const config = {
+  modelBasePath: 'file://models/',
+  backend: 'tensorflow',
+  debug: false,
+  videoOptimized: false,
+  async: false,
+  filter: {
+    enabled: true,
+  },
+  face: {
+    enabled: true,
+    detector: { enabled: true, rotation: true },
+    mesh: { enabled: true },
+    iris: { enabled: true },
+    description: { enabled: true },
+    emotion: { enabled: true },
+  },
+  hand: { enabled: true },
+  body: { enabled: true },
+  object: { enabled: true },
+};
+
+test(Human, config);

test/test-node-wasm.js

@@ -0,0 +1,27 @@
+const Human = require('../dist/human.node-wasm.js').default;
+const test = require('./test-main.js').test;
+
+const config = {
+  modelBasePath: 'http://localhost:10030/models/',
+  backend: 'wasm',
+  wasmPath: 'assets/',
+  debug: false,
+  videoOptimized: false,
+  async: false,
+  filter: {
+    enabled: true,
+  },
+  face: {
+    enabled: true,
+    detector: { enabled: true, rotation: true },
+    mesh: { enabled: true },
+    iris: { enabled: true },
+    description: { enabled: true },
+    emotion: { enabled: true },
+  },
+  hand: { enabled: true },
+  body: { enabled: true },
+  object: { enabled: false },
+};
+
+test(Human, config);

test/test-node.js

@@ -1,100 +1,26 @@
-const log = require('@vladmandic/pilogger');
-// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
-const tf = require('@tensorflow/tfjs-node');
 const Human = require('../dist/human.node.js').default;
+const test = require('./test-main.js').test;
 
 const config = {
+  modelBasePath: 'file://models/',
   backend: 'tensorflow',
   debug: false,
   videoOptimized: false,
   async: false,
-  warmup: 'full',
-  modelBasePath: 'file://models/',
   filter: {
     enabled: true,
   },
   face: {
     enabled: true,
-    detector: { enabled: true, rotation: false },
+    detector: { enabled: true, rotation: true },
     mesh: { enabled: true },
     iris: { enabled: true },
     description: { enabled: true },
     emotion: { enabled: true },
   },
-  hand: {
+  hand: { enabled: true },
-    enabled: true,
+  body: { enabled: true },
-  },
-  // body: { modelPath: 'efficientpose.json', enabled: true },
-  // body: { modelPath: 'blazepose.json', enabled: true },
-  body: { modelPath: 'posenet.json', enabled: true },
   object: { enabled: true },
 };
 
-async function testInstance(human) {
+test(Human, config);
-  if (human) log.state('passed: create human');
-  else log.error('failed: create human');
-
-  // if (!human.tf) human.tf = tf;
-  log.info('human version:', human.version);
-  log.info('tfjs version:', human.tf.version_core);
-  log.info('platform:', human.sysinfo.platform);
-  log.info('agent:', human.sysinfo.agent);
-
-  await human.load();
-  if (human.models) {
-    log.state('passed: load models');
-    const keys = Object.keys(human.models);
-    const loaded = keys.filter((model) => human.models[model]);
-    log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length);
-  } else {
-    log.error('failed: load models');
-  }
-
-  let warmup;
-  try {
-    warmup = await human.warmup();
-  } catch (err) {
-    log.error('error warmup');
-  }
-  if (warmup) {
-    log.state('passed: warmup:', config.warmup);
-    log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length);
-    log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total);
-  } else {
-    log.error('failed: warmup');
-  }
-  const random = tf.randomNormal([1, 1024, 1024, 3]);
-  let detect;
-  try {
-    detect = await human.detect(random);
-  } catch (err) {
-    log.error('error: detect', err);
-  }
-  tf.dispose(random);
-  if (detect) {
-    log.state('passed: detect:', 'random');
-    log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length);
-    log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
-  } else {
-    log.error('failed: detect');
-  }
-}
-
-async function test() {
-  log.info('testing instance#1 - none');
-  config.warmup = 'none';
-  const human1 = new Human(config);
-  await testInstance(human1);
-
-  log.info('testing instance#2 - face');
-  config.warmup = 'face';
-  const human2 = new Human(config);
-  await testInstance(human2);
-
-  log.info('testing instance#3 - body');
-  config.warmup = 'body';
-  const human3 = new Human(config);
-  await testInstance(human3);
-}
-
-test();

wiki

@@ -1 +1 @@
-Subproject commit 77b1cd6cfd86abe0b21aae23e2be2beff84b68ff
+Subproject commit 00a239fa51eda5b366f0e1d05d66fccf4edb0ce1