From e42465d0042b9490a9f06b0d444e128e6c735db6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 14 Apr 2021 12:53:00 -0400 Subject: [PATCH] update tfjs 3.4.0 --- .eslintrc.json | 1 + CHANGELOG.md | 3 + demo/helpers/webrtc.js | 1 - demo/index.js | 4 +- package.json | 24 +++--- src/human.ts | 40 +++++----- test/README.md | 47 ++++++++++++ test/test-main.js | 162 +++++++++++++++++++++++++++++++++++++++++ test/test-node-gpu.js | 26 +++++++ test/test-node-wasm.js | 27 +++++++ test/test-node.js | 86 ++-------------------- wiki | 2 +- 12 files changed, 310 insertions(+), 113 deletions(-) create mode 100644 test/README.md create mode 100644 test/test-main.js create mode 100644 test/test-node-gpu.js create mode 100644 test/test-node-wasm.js diff --git a/.eslintrc.json b/.eslintrc.json index dbcd90ea..9005ee77 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -61,6 +61,7 @@ "node/no-unpublished-import": "off", "node/no-unpublished-require": "off", "node/no-unsupported-features/es-syntax": "off", + "no-lonely-if": "off", "node/shebang": "off", "object-curly-newline": "off", "prefer-destructuring": "off", diff --git a/CHANGELOG.md b/CHANGELOG.md index c7f8b00c..d8b417b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/04/13 mandic00@live.com + + ### **1.5.1** 2021/04/13 mandic00@live.com - fix for safari imagebitmap diff --git a/demo/helpers/webrtc.js b/demo/helpers/webrtc.js index af99fbbb..56ccabd8 100644 --- a/demo/helpers/webrtc.js +++ b/demo/helpers/webrtc.js @@ -41,7 +41,6 @@ async function webRTC(server, streamName, elementName) { connection.ontrack = (event) => { stream.addTrack(event.track); const video = (typeof elementName === 'string') ? document.getElementById(elementName) : elementName; - // @ts-ignore if (video instanceof HTMLVideoElement) video.srcObject = stream; else log('element is not a video element:', elementName); // video.onloadeddata = async () => log('resolution:', video.videoWidth, video.videoHeight); diff --git a/demo/index.js b/demo/index.js index 45da4889..516014c8 100644 --- a/demo/index.js +++ b/demo/index.js @@ -198,7 +198,7 @@ async function setupCamera() { ui.busy = true; const viewportScale = Math.min(1, Math.round(100 * window.outerWidth / 700) / 100); // log('demo viewport scale:', viewportScale); - document.querySelector('meta[name=viewport]').setAttribute('content', `width=device-width, shrink-to-fit=no; initial-scale=${viewportScale}`); + document.querySelector('meta[name=viewport]').setAttribute('content', `width=device-width, shrink-to-fit=no, initial-scale=${viewportScale}`); const video = document.getElementById('video'); const canvas = document.getElementById('canvas'); const output = document.getElementById('log'); @@ -587,7 +587,7 @@ async function main() { log('overriding worker:', ui.useWorker); } if (params.has('backend')) { - userConfig.backend = JSON.parse(params.get('backend')); + userConfig.backend = params.get('backend'); log('overriding backend:', userConfig.backend); } if (params.has('preload')) { diff --git a/package.json b/package.json index c479a4c1..558d475c 100644 --- a/package.json +++ b/package.json @@ -51,20 +51,21 @@ "tensorflow" ], "devDependencies": { - "@tensorflow/tfjs": "^3.3.0", - "@tensorflow/tfjs-backend-cpu": "^3.3.0", - "@tensorflow/tfjs-backend-wasm": "^3.3.0", - "@tensorflow/tfjs-backend-webgl": "^3.3.0", - "@tensorflow/tfjs-converter": "^3.3.0", - "@tensorflow/tfjs-core": "^3.3.0", - "@tensorflow/tfjs-data": "^3.3.0", - "@tensorflow/tfjs-layers": "^3.3.0", - "@tensorflow/tfjs-node": "^3.3.0", - "@tensorflow/tfjs-node-gpu": "^3.3.0", + "@tensorflow/tfjs": "^3.4.0", + "@tensorflow/tfjs-backend-cpu": "^3.4.0", + "@tensorflow/tfjs-backend-wasm": "^3.4.0", + "@tensorflow/tfjs-backend-webgl": "^3.4.0", + "@tensorflow/tfjs-converter": "^3.4.0", + "@tensorflow/tfjs-core": "^3.4.0", + "@tensorflow/tfjs-data": "^3.4.0", + "@tensorflow/tfjs-layers": "^3.4.0", + "@tensorflow/tfjs-node": "^3.4.0", + "@tensorflow/tfjs-node-gpu": "^3.4.0", "@types/node": "^14.14.37", "@typescript-eslint/eslint-plugin": "^4.22.0", "@typescript-eslint/parser": "^4.22.0", "@vladmandic/pilogger": "^0.2.16", + "canvas": "^2.7.0", "chokidar": "^3.5.1", "dayjs": "^1.10.4", "esbuild": "^0.11.10", @@ -75,9 +76,10 @@ "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^5.1.0", "hint": "^6.1.3", + "node-fetch": "^2.6.1", "rimraf": "^3.0.2", "seedrandom": "^3.0.5", - "simple-git": "^2.37.0", + "simple-git": "^2.38.0", "tslib": "^2.2.0", "typedoc": "^0.20.35", "typescript": "^4.2.4" diff --git a/src/human.ts b/src/human.ts index e52f2c2f..96b142f1 100644 --- a/src/human.ts +++ b/src/human.ts @@ -157,7 +157,7 @@ export class Human { faceres: null, }; // export access to image processing - // @ts-ignore // typescript cannot infer type + // @ts-ignore eslint-typescript cannot correctly infer type in anonymous function this.image = (input: Input) => image.process(input, this.config); // export raw access to underlying models this.classes = { @@ -266,9 +266,7 @@ export class Human { this.models.gender, this.models.emotion, this.models.embedding, - // @ts-ignore // typescript cannot infer type this.models.handpose, - // @ts-ignore // typescript cannot infer type this.models.posenet, this.models.blazepose, this.models.efficientpose, @@ -280,8 +278,8 @@ export class Human { this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null), this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null), this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null), - this.models.handpose || (this.config.hand.enabled ? >handpose.load(this.config) : null), - this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null), + this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null), + this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null), this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null), this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null), @@ -340,7 +338,7 @@ export class Human { const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT'); if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`); - if (!simd) log('warning: wasm simd support is not enabled'); + if (this.config.debug && !simd) log('warning: wasm simd support is not enabled'); } if (this.config.backend === 'humangl') backend.register(); @@ -574,23 +572,29 @@ export class Human { /** @hidden */ #warmupNode = async () => { - // @ts-ignore - if (typeof tf.node === 'undefined') { - if (this.config.debug) log('Warmup tfjs-node not loaded'); - return null; - } const atob = (str) => Buffer.from(str, 'base64'); let img; if (this.config.warmup === 'face') img = atob(sample.face); if (this.config.warmup === 'body' || this.config.warmup === 'full') img = atob(sample.body); if (!img) return null; - // @ts-ignore // tf.node is only defined when compiling for nodejs - const data = tf.node?.decodeJpeg(img); - const expanded = data.expandDims(0); - this.tf.dispose(data); - // log('Input:', expanded); - const res = await this.detect(expanded, this.config); - this.tf.dispose(expanded); + let res; + if (typeof tf['node'] !== 'undefined') { + const data = tf['node'].decodeJpeg(img); + const expanded = data.expandDims(0); + this.tf.dispose(data); + // log('Input:', expanded); + res = await this.detect(expanded, this.config); + this.tf.dispose(expanded); + } else { + if (this.config.debug) log('Warmup tfjs-node not loaded'); + /* + const input = await canvasJS.loadImage(img); + const canvas = canvasJS.createCanvas(input.width, input.height); + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0, input.width, input.height); + res = await this.detect(input, this.config); + */ + } return res; } diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..74922714 --- /dev/null +++ b/test/README.md @@ -0,0 +1,47 @@ +# Test Results + +## Automatic Tests + +### NodeJS using TensorFlow library + +- Face rotation is disabled for `NodeJS` platform: + `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'` + +- Image filters are disabled due to lack of Canvas and WeBGL access + +### NodeJS with GPU acceleation using CUDA + +- Face rotation is disabled for `NodeJS` platform: + `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'` + +- Image filters are disabled due to lack of Canvas and WeBGL access + +### NodeJS using WASM + +- Requires dev http server + See +- Only supported input is Tensor due to missing image decoders +- Warmup returns null and is marked as failed + Missing native Image implementation in NodeJS +- Fails on object detection: + `Kernel 'SparseToDense' not registered for backend 'wasm'` + + +
+ +## Manual Tests + +### Browser using WebGL backend + +- Chrome/Edge: All Passing +- Firefox: WebWorkers not supported due to missing support for OffscreenCanvas +- Safari: Limited Testing + +### Browser using WASM backend + +- Chrome/Edge: All Passing +- Firefox: WebWorkers not supported due to missing support for OffscreenCanvas +- Safari: Limited Testing +- Fails on object detection: + `Kernel 'SparseToDense' not registered for backend 'wasm'` + diff --git a/test/test-main.js b/test/test-main.js new file mode 100644 index 00000000..c447695a --- /dev/null +++ b/test/test-main.js @@ -0,0 +1,162 @@ +const process = require('process'); +const path = require('path'); +const log = require('@vladmandic/pilogger'); +const canvasJS = require('canvas'); +const fetch = require('node-fetch').default; + +let config; +log.info('test:', path.basename(process.argv[1])); + +async function testHTTP() { + if (config.modelBasePath.startsWith('file:')) return true; + return new Promise((resolve) => { + fetch(config.modelBasePath) + .then((res) => { + if (res && res.ok) log.state('passed: model server:', config.modelBasePath); + else log.error('failed: model server:', config.modelBasePath); + resolve(res && res.ok); + }) + .catch((err) => { + log.error('failed: model server:', err.message); + resolve(false); + }); + }); +} + +async function getImage(human, input) { + let img; + try { + img = await canvasJS.loadImage(input); + } catch (err) { + log.error('failed: load image', input, err.message); + return img; + } + const canvas = canvasJS.createCanvas(img.width, img.height); + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0, img.width, img.height); + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const res = human.tf.tidy(() => { + const tensor = human.tf.tensor(Array.from(imageData.data), [canvas.height, canvas.width, 4], 'int32'); // create rgba image tensor from flat array + const channels = human.tf.split(tensor, 4, 2); // split rgba to channels + const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb + const reshape = human.tf.reshape(rgb, [1, canvas.width, canvas.height, 3]); // move extra dim from the end of tensor and use it as batch number instead + return reshape; + }); + if (res && res.shape[0] === 1 && res.shape[3] === 3) log.state('passed: load image:', input, res.shape); + else log.error('failed: load image:', input, res); + return res; +} + +function printResults(detect) { + const person = (detect.face && detect.face[0]) ? { confidence: detect.face[0].confidence, age: detect.face[0].age, gender: detect.face[0].gender } : {}; + const object = (detect.object && detect.object[0]) ? { score: detect.object[0].score, class: detect.object[0].label } : {}; + const body = (detect.body && detect.body[0]) ? { score: detect.body[0].score, keypoints: detect.body[0].keypoints.length } : {}; + if (detect.face) log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length, person, object, body); + if (detect.performance) log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total); +} + +async function testInstance(human) { + if (human) log.state('passed: create human'); + else log.error('failed: create human'); + + // if (!human.tf) human.tf = tf; + log.info('human version:', human.version); + log.info('platform:', human.sysinfo.platform, 'agent:', human.sysinfo.agent); + log.info('tfjs version:', human.tf.version.tfjs); + + await human.load(); + if (config.backend === human.tf.getBackend()) log.state('passed: set backend:', config.backend); + else log.error('failed: set backend:', config.backend); + + if (human.models) { + log.state('passed: load models'); + const keys = Object.keys(human.models); + const loaded = keys.filter((model) => human.models[model]); + log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length); + return true; + } + log.error('failed: load models'); + return false; +} + +async function testWarmup(human, title) { + let warmup; + try { + warmup = await human.warmup(config); + } catch (err) { + log.error('error warmup'); + } + if (warmup) { + log.state('passed: warmup:', config.warmup, title); + printResults(warmup); + return true; + } + log.error('failed: warmup:', config.warmup, title); + return false; +} + +async function testDetect(human, input, title) { + const image = input ? await getImage(human, input) : human.tf.randomNormal([1, 1024, 1024, 3]); + if (!image) { + log.error('failed: detect: input is null'); + return false; + } + let detect; + try { + detect = await human.detect(image, config); + } catch (err) { + log.error('error: detect', err); + } + if (image instanceof human.tf.Tensor) human.tf.dispose(image); + if (detect) { + log.state('passed: detect:', input || 'random', title); + printResults(detect); + return true; + } + log.error('failed: detect', input || 'random', title); + return false; +} + +async function test(Human, inputConfig) { + config = inputConfig; + const ok = await testHTTP(); + if (!ok) { + log.warn('aborting test'); + return; + } + const t0 = process.hrtime.bigint(); + const human = new Human(config); + await testInstance(human); + config.warmup = 'none'; + await testWarmup(human, 'default'); + config.warmup = 'face'; + await testWarmup(human, 'default'); + config.warmup = 'body'; + await testWarmup(human, 'default'); + + log.info('test body variants'); + config.body = { modelPath: 'posenet.json', enabled: true }; + await testDetect(human, 'assets/human-sample-body.jpg', 'posenet'); + config.body = { modelPath: 'efficientpose.json', enabled: true }; + await testDetect(human, 'assets/human-sample-body.jpg', 'efficientpose'); + config.body = { modelPath: 'blazepose.json', enabled: true }; + await testDetect(human, 'assets/human-sample-body.jpg', 'blazepose'); + + await testDetect(human, null, 'default'); + log.info('test: first instance'); + await testDetect(human, 'assets/sample-me.jpg', 'default'); + log.info('test: second instance'); + const second = new Human(config); + await testDetect(second, 'assets/sample-me.jpg', 'default'); + log.info('test: concurrent'); + await Promise.all([ + testDetect(human, 'assets/human-sample-face.jpg', 'default'), + testDetect(second, 'assets/human-sample-face.jpg', 'default'), + testDetect(human, 'assets/human-sample-body.jpg', 'default'), + testDetect(second, 'assets/human-sample-body.jpg', 'default'), + ]); + const t1 = process.hrtime.bigint(); + log.info('test complete:', Math.trunc(parseInt((t1 - t0).toString()) / 1000 / 1000), 'ms'); +} + +exports.test = test; diff --git a/test/test-node-gpu.js b/test/test-node-gpu.js new file mode 100644 index 00000000..b8bf175f --- /dev/null +++ b/test/test-node-gpu.js @@ -0,0 +1,26 @@ +const Human = require('../dist/human.node-gpu.js').default; +const test = require('./test-main.js').test; + +const config = { + modelBasePath: 'file://models/', + backend: 'tensorflow', + debug: false, + videoOptimized: false, + async: false, + filter: { + enabled: true, + }, + face: { + enabled: true, + detector: { enabled: true, rotation: true }, + mesh: { enabled: true }, + iris: { enabled: true }, + description: { enabled: true }, + emotion: { enabled: true }, + }, + hand: { enabled: true }, + body: { enabled: true }, + object: { enabled: true }, +}; + +test(Human, config); diff --git a/test/test-node-wasm.js b/test/test-node-wasm.js new file mode 100644 index 00000000..8400814c --- /dev/null +++ b/test/test-node-wasm.js @@ -0,0 +1,27 @@ +const Human = require('../dist/human.node-wasm.js').default; +const test = require('./test-main.js').test; + +const config = { + modelBasePath: 'http://localhost:10030/models/', + backend: 'wasm', + wasmPath: 'assets/', + debug: false, + videoOptimized: false, + async: false, + filter: { + enabled: true, + }, + face: { + enabled: true, + detector: { enabled: true, rotation: true }, + mesh: { enabled: true }, + iris: { enabled: true }, + description: { enabled: true }, + emotion: { enabled: true }, + }, + hand: { enabled: true }, + body: { enabled: true }, + object: { enabled: false }, +}; + +test(Human, config); diff --git a/test/test-node.js b/test/test-node.js index 23fd14bc..6fea955a 100644 --- a/test/test-node.js +++ b/test/test-node.js @@ -1,100 +1,26 @@ -const log = require('@vladmandic/pilogger'); -// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const tf = require('@tensorflow/tfjs-node'); const Human = require('../dist/human.node.js').default; +const test = require('./test-main.js').test; const config = { + modelBasePath: 'file://models/', backend: 'tensorflow', debug: false, videoOptimized: false, async: false, - warmup: 'full', - modelBasePath: 'file://models/', filter: { enabled: true, }, face: { enabled: true, - detector: { enabled: true, rotation: false }, + detector: { enabled: true, rotation: true }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, }, - hand: { - enabled: true, - }, - // body: { modelPath: 'efficientpose.json', enabled: true }, - // body: { modelPath: 'blazepose.json', enabled: true }, - body: { modelPath: 'posenet.json', enabled: true }, + hand: { enabled: true }, + body: { enabled: true }, object: { enabled: true }, }; -async function testInstance(human) { - if (human) log.state('passed: create human'); - else log.error('failed: create human'); - - // if (!human.tf) human.tf = tf; - log.info('human version:', human.version); - log.info('tfjs version:', human.tf.version_core); - log.info('platform:', human.sysinfo.platform); - log.info('agent:', human.sysinfo.agent); - - await human.load(); - if (human.models) { - log.state('passed: load models'); - const keys = Object.keys(human.models); - const loaded = keys.filter((model) => human.models[model]); - log.data(' result: defined models:', keys.length, 'loaded models:', loaded.length); - } else { - log.error('failed: load models'); - } - - let warmup; - try { - warmup = await human.warmup(); - } catch (err) { - log.error('error warmup'); - } - if (warmup) { - log.state('passed: warmup:', config.warmup); - log.data(' result: face:', warmup.face?.length, 'body:', warmup.body?.length, 'hand:', warmup.hand?.length, 'gesture:', warmup.gesture?.length, 'object:', warmup.object?.length); - log.data(' result: performance:', 'load:', warmup.performance?.load, 'total:', warmup.performance?.total); - } else { - log.error('failed: warmup'); - } - const random = tf.randomNormal([1, 1024, 1024, 3]); - let detect; - try { - detect = await human.detect(random); - } catch (err) { - log.error('error: detect', err); - } - tf.dispose(random); - if (detect) { - log.state('passed: detect:', 'random'); - log.data(' result: face:', detect.face?.length, 'body:', detect.body?.length, 'hand:', detect.hand?.length, 'gesture:', detect.gesture?.length, 'object:', detect.object?.length); - log.data(' result: performance:', 'load:', detect?.performance.load, 'total:', detect.performance?.total); - } else { - log.error('failed: detect'); - } -} - -async function test() { - log.info('testing instance#1 - none'); - config.warmup = 'none'; - const human1 = new Human(config); - await testInstance(human1); - - log.info('testing instance#2 - face'); - config.warmup = 'face'; - const human2 = new Human(config); - await testInstance(human2); - - log.info('testing instance#3 - body'); - config.warmup = 'body'; - const human3 = new Human(config); - await testInstance(human3); -} - -test(); +test(Human, config); diff --git a/wiki b/wiki index 77b1cd6c..00a239fa 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 77b1cd6cfd86abe0b21aae23e2be2beff84b68ff +Subproject commit 00a239fa51eda5b366f0e1d05d66fccf4edb0ce1