From 525634ad260b2554b90c6332fa10af8c465b14e3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Jun 2021 17:51:46 -0400 Subject: [PATCH] modularize build platform --- CHANGELOG.md | 4 +--- demo/facematch.js | 5 ++--- demo/index.js | 6 +++--- src/config.ts | 13 +++++++++---- src/faceres/faceres.ts | 2 +- src/human.ts | 38 +++++++++++++++++++++----------------- src/object/nanodet.ts | 2 +- test/test-node-gpu.js | 5 ++--- test/test-node-wasm.js | 5 ++--- test/test-node.js | 5 ++--- tsconfig.json | 18 ++++++++++++++---- wiki | 2 +- 12 files changed, 59 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 490d4f2f..25eca82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,7 @@ Repository: **** ### **HEAD -> main** 2021/06/05 mandic00@live.com - -### **origin/main** 2021/06/05 mandic00@live.com - +- minor git corruption - unified build - enable body segmentation and background replacement - work on body segmentation diff --git a/demo/facematch.js b/demo/facematch.js index dcaffc78..cd4d2982 100644 --- a/demo/facematch.js +++ b/demo/facematch.js @@ -27,9 +27,8 @@ const userConfig = { hand: { enabled: false }, gesture: { enabled: false }, body: { enabled: false }, - filter: { - enabled: false, - }, + filter: { enabled: true }, + segmentation: { enabled: false }, }; const human = new Human(userConfig); // new instance of human diff --git a/demo/index.js b/demo/index.js index c3027418..c6203fdd 100644 --- a/demo/index.js +++ b/demo/index.js @@ -31,6 +31,7 @@ let userConfig = { warmup: 'none', backend: 'humangl', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/', + segmentation: { enabled: true }, /* async: false, cacheSensitivity: 0, @@ -210,10 +211,9 @@ async function drawResults(input) { // draw fps chart await menu.process.updateChart('FPS', ui.detectFPS); - // get updated canvas if missing or if we want buffering, but skip if segmentation is enabled - if (userConfig.segmentation.enabled) { + if (userConfig.segmentation.enabled && ui.buffered) { // refresh segmentation if using buffered output result.canvas = await human.segmentation(input, ui.background, userConfig); - } else if (!result.canvas || ui.buffered) { + } else if (!result.canvas || ui.buffered) { // refresh with input if using buffered output or if missing canvas const image = await human.image(input); result.canvas = image.canvas; human.tf.dispose(image.tensor); diff --git a/src/config.ts b/src/config.ts index 843d6540..96e8da70 100644 --- a/src/config.ts +++ b/src/config.ts @@ -198,7 +198,10 @@ export interface Config { }, /** Controlls and configures all body segmentation module - * if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output + * removes background from input containing person + * if segmentation is enabled it will run as preprocessing task before any other model + * alternatively leave it disabled and use it on-demand using human.segmentation method which can + * remove background or replace it with user-provided background * * - enabled: true/false * - modelPath: object detection model, can be absolute path or relative to modelBasePath @@ -351,9 +354,11 @@ const config: Config = { }, segmentation: { - enabled: false, // if segmentation is enabled, output result.canvas will be augmented - // with masked image containing only person output - // segmentation is not triggered as part of detection and requires separate call to human.segmentation + enabled: false, // controlls and configures all body segmentation module + // removes background from input containing person + // if segmentation is enabled it will run as preprocessing task before any other model + // alternatively leave it disabled and use it on-demand using human.segmentation method which can + // remove background or replace it with user-provided background modelPath: 'selfie.json', // experimental: object detection model, can be absolute path or relative to modelBasePath // can be 'selfie' or 'meet' }, diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 9a74f8fe..766ad9d8 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -39,7 +39,7 @@ export function similarity(embedding1: Array, embedding2: Array, if (embedding1?.length !== embedding2?.length) return 0; // general minkowski distance, euclidean distance is limited case where order is 2 const distance = 5.0 * embedding1 - .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared + .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared .reduce((sum, now) => (sum + now), 0) // sum all distances ** (1 / order); // get root of const res = Math.max(0, 100 - distance) / 100.0; diff --git a/src/human.ts b/src/human.ts index cd73d682..3864a684 100644 --- a/src/human.ts +++ b/src/human.ts @@ -435,6 +435,7 @@ export class Human { return new Promise(async (resolve) => { this.state = 'config'; let timeStamp; + let elapsedTime; // update configuration this.config = mergeDeep(this.config, userConfig) as Config; @@ -473,14 +474,31 @@ export class Human { */ timeStamp = now(); - const process = image.process(input, this.config); + let process = image.process(input, this.config); + this.performance.image = Math.trunc(now() - timeStamp); + this.analyze('Get Image:'); + + // run segmentation preprocessing + if (this.config.segmentation.enabled && process && process.tensor) { + this.analyze('Start Segmentation:'); + this.state = 'run:segmentation'; + timeStamp = now(); + await segmentation.predict(process); + elapsedTime = Math.trunc(now() - timeStamp); + if (elapsedTime > 0) this.performance.segmentation = elapsedTime; + if (process.canvas) { + // replace input + process.tensor.dispose(); + process = image.process(process.canvas, this.config); + } + this.analyze('End Segmentation:'); + } + if (!process || !process.tensor) { log('could not convert input to tensor'); resolve({ error: 'could not convert input to tensor' }); return; } - this.performance.image = Math.trunc(now() - timeStamp); - this.analyze('Get Image:'); timeStamp = now(); this.config.skipFrame = await this.#skipFrame(process.tensor); @@ -497,7 +515,6 @@ export class Human { let bodyRes; let handRes; let objectRes; - let elapsedTime; // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion if (this.config.async) { @@ -573,19 +590,6 @@ export class Human { else if (this.performance.gesture) delete this.performance.gesture; } - // run segmentation - /* not triggered as part of detect - if (this.config.segmentation.enabled) { - this.analyze('Start Segmentation:'); - this.state = 'run:segmentation'; - timeStamp = now(); - await segmentation.predict(process, this.config); - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.segmentation = elapsedTime; - this.analyze('End Segmentation:'); - } - */ - this.performance.total = Math.trunc(now() - timeStart); this.state = 'idle'; this.result = { diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 01a148fa..54a2a21d 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -96,7 +96,7 @@ async function process(res, inputSize, outputShape, config) { // filter & sort results results = results - .filter((a, idx) => nmsIdx.includes(idx)) + .filter((_val, idx) => nmsIdx.includes(idx)) .sort((a, b) => (b.score - a.score)); return results; diff --git a/test/test-node-gpu.js b/test/test-node-gpu.js index 9b5d83e5..eb3c8ed1 100644 --- a/test/test-node-gpu.js +++ b/test/test-node-gpu.js @@ -6,9 +6,6 @@ const config = { backend: 'tensorflow', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -20,6 +17,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: true }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/test/test-node-wasm.js b/test/test-node-wasm.js index d154da57..14bf3567 100644 --- a/test/test-node-wasm.js +++ b/test/test-node-wasm.js @@ -8,9 +8,6 @@ const config = { // wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -22,6 +19,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: false }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/test/test-node.js b/test/test-node.js index 169ac997..538445f1 100644 --- a/test/test-node.js +++ b/test/test-node.js @@ -6,9 +6,6 @@ const config = { backend: 'tensorflow', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -20,6 +17,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: true }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/tsconfig.json b/tsconfig.json index 1e4fb9b8..9c034389 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,17 +7,16 @@ "typeRoots": ["node_modules/@types"], "outDir": "types", "declaration": true, + "allowSyntheticDefaultImports": true, "emitDeclarationOnly": true, "emitDecoratorMetadata": true, "experimentalDecorators": true, "importHelpers": true, - "noImplicitAny": false, "preserveConstEnums": true, "removeComments": false, "resolveJsonModule": true, "skipLibCheck": true, - "sourceMap": false, - "strictNullChecks": true, + "sourceMap": true, "allowJs": true, "baseUrl": "./", "paths": { @@ -25,10 +24,21 @@ "@tensorflow/tfjs-node/dist/io/file_system": ["node_modules/@tensorflow/tfjs-node/dist/io/file_system.js"], "@tensorflow/tfjs-core/dist/index": ["node_modules/@tensorflow/tfjs-core/dist/index.js"], "@tensorflow/tfjs-converter/dist/index": ["node_modules/@tensorflow/tfjs-converter/dist/index.js"] - } + }, + "strictNullChecks": true, + "noImplicitAny": false, + "noUnusedLocals": false, + "noImplicitReturns": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedParameters": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "allowUnreachableCode": false }, "formatCodeOptions": { "indentSize": 2, "tabSize": 2 }, "include": ["src/*", "src/***/*"], + "exclude": ["node_modules/"], "typedocOptions": { "excludePrivate": true, "excludeExternals": true, diff --git a/wiki b/wiki index c9408224..9e92e5ee 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c9408224d824368facc264c00e05d7b520d69051 +Subproject commit 9e92e5eec1e60b5ea58dbf1c4bbc67c828bcf673