experimental node-wasm support

pull/134/head
Vladimir Mandic 2021-04-13 21:45:45 -04:00
parent c75abab0af
commit ad99ed2122
20 changed files with 1036 additions and 154 deletions

View File

@ -1,6 +1,6 @@
# @vladmandic/human # @vladmandic/human
Version: **1.5.0** Version: **1.5.1**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>** Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,8 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog ## Changelog
### **HEAD -> main** 2021/04/13 mandic00@live.com ### **1.5.1** 2021/04/13 mandic00@live.com
- fix for safari imagebitmap
- refactored human.config and human.draw
### **1.4.3** 2021/04/12 mandic00@live.com ### **1.4.3** 2021/04/12 mandic00@live.com

10
TODO.md
View File

@ -2,19 +2,15 @@
## Big Ticket Items ## Big Ticket Items
- Strong(er) typing - Improve automated testing framework
- Automated testing framework
- TypeDoc comments
## Explore Models ## Explore Models
- EfficientPose
<https://github.com/daniegr/EfficientPose>
<https://github.com/PINTO0309/PINTO_model_zoo/tree/main/084_EfficientPose>
- InsightFace - InsightFace
RetinaFace detetor and ArcFace recognition RetinaFace detector and ArcFace recognition
<https://github.com/deepinsight/insightface> <https://github.com/deepinsight/insightface>
## Issues ## Issues
- box sizing on mobile - box sizing on mobile
- canvas.js for wasm on node

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

104
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

104
dist/human.js vendored

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

735
dist/human.node-wasm.js vendored Normal file

File diff suppressed because one or more lines are too long

7
dist/human.node-wasm.js.map vendored Normal file

File diff suppressed because one or more lines are too long

14
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -24,7 +24,7 @@
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js", "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js",
"dev": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/serve.js", "dev": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/serve.js",
"build": "rimraf dist/* typedoc/* types/* && node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/build.js", "build": "rimraf dist/* typedoc/* types/* && node --trace-warnings --unhandled-rejections=strict --trace-uncaught server/build.js",
"lint": "eslint src server demo", "lint": "eslint src server demo test",
"test": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation test/test-node.js" "test": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation test/test-node.js"
}, },
"keywords": [ "keywords": [

View File

@ -99,6 +99,22 @@ const targets = {
external: ['@tensorflow'], external: ['@tensorflow'],
}, },
}, },
nodeWASM: {
tfjs: {
platform: 'node',
format: 'cjs',
entryPoints: ['src/tfjs/tf-node-wasm.ts'],
outfile: 'dist/tfjs.esm.js',
external: ['@tensorflow'],
},
node: {
platform: 'node',
format: 'cjs',
entryPoints: ['src/human.ts'],
outfile: 'dist/human.node-wasm.js',
external: ['@tensorflow'],
},
},
browserNoBundle: { browserNoBundle: {
tfjs: { tfjs: {

View File

@ -329,13 +329,14 @@ export class Human {
if (this.config.backend && this.config.backend.length > 0) { if (this.config.backend && this.config.backend.length > 0) {
// force browser vs node backend // force browser vs node backend
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl'; if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl';
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'wasm')) this.config.backend = 'tensorflow'; if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) this.config.backend = 'tensorflow';
if (this.config.debug) log('setting backend:', this.config.backend); if (this.config.debug) log('setting backend:', this.config.backend);
if (this.config.backend === 'wasm') { if (this.config.backend === 'wasm') {
if (this.config.debug) log('wasm path:', this.config.wasmPath); if (this.config.debug) log('wasm path:', this.config.wasmPath);
this.tf.setWasmPaths(this.config.wasmPath); if (typeof this.tf?.setWasmPaths !== 'undefined') this.tf.setWasmPaths(this.config.wasmPath);
else throw new Error('Human: WASM backend is not loaded');
const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT'); const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`); if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`);
@ -573,8 +574,16 @@ export class Human {
/** @hidden */ /** @hidden */
#warmupNode = async () => { #warmupNode = async () => {
// @ts-ignore
if (typeof tf.node === 'undefined') {
if (this.config.debug) log('Warmup tfjs-node not loaded');
return null;
}
const atob = (str) => Buffer.from(str, 'base64'); const atob = (str) => Buffer.from(str, 'base64');
const img = this.config.warmup === 'face' ? atob(sample.face) : atob(sample.body); let img;
if (this.config.warmup === 'face') img = atob(sample.face);
if (this.config.warmup === 'body' || this.config.warmup === 'full') img = atob(sample.body);
if (!img) return null;
// @ts-ignore // tf.node is only defined when compiling for nodejs // @ts-ignore // tf.node is only defined when compiling for nodejs
const data = tf.node?.decodeJpeg(img); const data = tf.node?.decodeJpeg(img);
const expanded = data.expandDims(0); const expanded = data.expandDims(0);
@ -592,6 +601,7 @@ export class Human {
async warmup(userConfig: Config | Object = {}): Promise<Result | { error }> { async warmup(userConfig: Config | Object = {}): Promise<Result | { error }> {
const t0 = now(); const t0 = now();
if (userConfig) this.config = mergeDeep(this.config, userConfig); if (userConfig) this.config = mergeDeep(this.config, userConfig);
if (!this.config.warmup || this.config.warmup === 'none') return { error: 'null' };
const save = this.config.videoOptimized; const save = this.config.videoOptimized;
this.config.videoOptimized = false; this.config.videoOptimized = false;
let res; let res;

2
src/tfjs/tf-node-wasm.ts Normal file
View File

@ -0,0 +1,2 @@
export * from '@tensorflow/tfjs';
export * from '@tensorflow/tfjs-backend-wasm';

View File

@ -50,36 +50,51 @@ async function testInstance(human) {
log.error('failed: load models'); log.error('failed: load models');
} }
const warmup = await human.warmup(); let warmup;
try {
warmup = await human.warmup();
} catch (err) {
log.error('error warmup');
}
if (warmup) { if (warmup) {
log.state('passed: warmup:', config.warmup); log.state('passed: warmup:', config.warmup);
log.data(' result: face:', warmup.face.length, 'body:', warmup.body.length, 'hand:', warmup.hand.length, 'gesture:', warmup.gesture.length, 'object:', warmup.object.length); log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length);
log.data(' result: performance:', 'load:', warmup.performance.load, 'total:', warmup.performance.total); log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total);
} else { } else {
log.error('failed: warmup'); log.error('failed: warmup');
} }
const random = tf.randomNormal([1, 1024, 1024, 3]); const random = tf.randomNormal([1, 1024, 1024, 3]);
const detect = await human.detect(random); let detect;
try {
detect = await human.detect(random);
} catch (err) {
log.error('error: detect', err);
}
tf.dispose(random); tf.dispose(random);
if (detect) { if (detect) {
log.state('passed: detect:', 'random'); log.state('passed: detect:', 'random');
log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length); log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length);
log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total); log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
} else { } else {
log.error('failed: detect'); log.error('failed: detect');
} }
} }
async function test() { async function test() {
log.info('testing instance#1'); log.info('testing instance#1 - none');
config.warmup = 'face'; config.warmup = 'none';
const human1 = new Human(config); const human1 = new Human(config);
await testInstance(human1); await testInstance(human1);
log.info('testing instance#2'); log.info('testing instance#2 - face');
config.warmup = 'body'; config.warmup = 'face';
const human2 = new Human(config); const human2 = new Human(config);
await testInstance(human2); await testInstance(human2);
log.info('testing instance#3 - body');
config.warmup = 'body';
const human3 = new Human(config);
await testInstance(human3);
} }
test(); test();

99
test/test-wasm.js Normal file
View File

@ -0,0 +1,99 @@
const log = require('@vladmandic/pilogger');
const tf = require('@tensorflow/tfjs');
const Human = require('../dist/human.node-wasm.js').default;
const config = {
backend: 'wasm',
wasmPath: 'assets/',
debug: false,
videoOptimized: false,
async: false,
modelBasePath: 'http://localhost:10030/models',
filter: {
enabled: true,
},
face: {
enabled: true,
detector: { enabled: true, rotation: false },
mesh: { enabled: true },
iris: { enabled: true },
description: { enabled: true },
emotion: { enabled: true },
},
hand: {
enabled: true,
},
// body: { modelPath: 'efficientpose.json', enabled: true },
// body: { modelPath: 'blazepose.json', enabled: true },
body: { modelPath: 'posenet.json', enabled: true },
object: { enabled: false }, // Error: Kernel 'SparseToDense' not registered for backend 'wasm'
};
async function testInstance(human) {
if (human) log.state('passed: create human');
else log.error('failed: create human');
// if (!human.tf) human.tf = tf;
log.info('human version:', human.version);
log.info('tfjs version:', human.tf.version_core);
log.info('platform:', human.sysinfo.platform);
log.info('agent:', human.sysinfo.agent);
await human.load();
if (human.models) {
log.state('passed: load models');
const keys = Object.keys(human.models);
const loaded = keys.filter((model) => human.models[model]);
log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length);
} else {
log.error('failed: load models');
}
let warmup;
try {
warmup = await human.warmup();
} catch (err) {
log.error('error warmup');
}
if (warmup) {
log.state('passed: warmup:', config.warmup);
log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length);
log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total);
} else {
log.error('failed: warmup');
}
const random = tf.randomNormal([1, 1024, 1024, 3]);
let detect;
try {
detect = await human.detect(random);
} catch (err) {
log.error('error: detect', err);
}
tf.dispose(random);
if (detect) {
log.state('passed: detect:', 'random');
log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length);
log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total);
} else {
log.error('failed: detect');
}
}
async function test() {
log.info('testing instance#1 - none');
config.warmup = 'none';
const human1 = new Human(config);
await testInstance(human1);
log.info('testing instance#2 - face');
config.warmup = 'face';
const human2 = new Human(config);
await testInstance(human2);
log.info('testing instance#3 - body');
config.warmup = 'body';
const human3 = new Human(config);
await testInstance(human3);
}
test();