From 7f5e5aa00a3324d2c1cc804333969fff73ff158c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 5 Jun 2021 17:51:46 -0400 Subject: [PATCH] modularize build platform --- CHANGELOG.md | 4 +--- demo/facematch.js | 5 ++-- demo/index.js | 6 ++--- src/config.ts | 13 ++++++---- src/faceres/faceres.ts | 2 +- src/human.ts | 38 ++++++++++++++++------------- src/object/nanodet.ts | 2 +- src/segmentation/segmentation.ts | 41 ++++++++++++++++---------------- test/test-node-gpu.js | 5 ++-- test/test-node-wasm.js | 5 ++-- test/test-node.js | 5 ++-- tsconfig.json | 18 ++++++++++---- wiki | 2 +- 13 files changed, 79 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 490d4f2f..25eca82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,7 @@ Repository: **** ### **HEAD -> main** 2021/06/05 mandic00@live.com - -### **origin/main** 2021/06/05 mandic00@live.com - +- minor git corruption - unified build - enable body segmentation and background replacement - work on body segmentation diff --git a/demo/facematch.js b/demo/facematch.js index dcaffc78..cd4d2982 100644 --- a/demo/facematch.js +++ b/demo/facematch.js @@ -27,9 +27,8 @@ const userConfig = { hand: { enabled: false }, gesture: { enabled: false }, body: { enabled: false }, - filter: { - enabled: false, - }, + filter: { enabled: true }, + segmentation: { enabled: false }, }; const human = new Human(userConfig); // new instance of human diff --git a/demo/index.js b/demo/index.js index c3027418..c6203fdd 100644 --- a/demo/index.js +++ b/demo/index.js @@ -31,6 +31,7 @@ let userConfig = { warmup: 'none', backend: 'humangl', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/', + segmentation: { enabled: true }, /* async: false, cacheSensitivity: 0, @@ -210,10 +211,9 @@ async function drawResults(input) { // draw fps chart await menu.process.updateChart('FPS', ui.detectFPS); - // get updated canvas if missing or if we want buffering, but skip if segmentation is enabled - if (userConfig.segmentation.enabled) { + if (userConfig.segmentation.enabled && ui.buffered) { // refresh segmentation if using buffered output result.canvas = await human.segmentation(input, ui.background, userConfig); - } else if (!result.canvas || ui.buffered) { + } else if (!result.canvas || ui.buffered) { // refresh with input if using buffered output or if missing canvas const image = await human.image(input); result.canvas = image.canvas; human.tf.dispose(image.tensor); diff --git a/src/config.ts b/src/config.ts index 843d6540..96e8da70 100644 --- a/src/config.ts +++ b/src/config.ts @@ -198,7 +198,10 @@ export interface Config { }, /** Controlls and configures all body segmentation module - * if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output + * removes background from input containing person + * if segmentation is enabled it will run as preprocessing task before any other model + * alternatively leave it disabled and use it on-demand using human.segmentation method which can + * remove background or replace it with user-provided background * * - enabled: true/false * - modelPath: object detection model, can be absolute path or relative to modelBasePath @@ -351,9 +354,11 @@ const config: Config = { }, segmentation: { - enabled: false, // if segmentation is enabled, output result.canvas will be augmented - // with masked image containing only person output - // segmentation is not triggered as part of detection and requires separate call to human.segmentation + enabled: false, // controlls and configures all body segmentation module + // removes background from input containing person + // if segmentation is enabled it will run as preprocessing task before any other model + // alternatively leave it disabled and use it on-demand using human.segmentation method which can + // remove background or replace it with user-provided background modelPath: 'selfie.json', // experimental: object detection model, can be absolute path or relative to modelBasePath // can be 'selfie' or 'meet' }, diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 9a74f8fe..766ad9d8 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -39,7 +39,7 @@ export function similarity(embedding1: Array, embedding2: Array, if (embedding1?.length !== embedding2?.length) return 0; // general minkowski distance, euclidean distance is limited case where order is 2 const distance = 5.0 * embedding1 - .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared + .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared .reduce((sum, now) => (sum + now), 0) // sum all distances ** (1 / order); // get root of const res = Math.max(0, 100 - distance) / 100.0; diff --git a/src/human.ts b/src/human.ts index cd73d682..3864a684 100644 --- a/src/human.ts +++ b/src/human.ts @@ -435,6 +435,7 @@ export class Human { return new Promise(async (resolve) => { this.state = 'config'; let timeStamp; + let elapsedTime; // update configuration this.config = mergeDeep(this.config, userConfig) as Config; @@ -473,14 +474,31 @@ export class Human { */ timeStamp = now(); - const process = image.process(input, this.config); + let process = image.process(input, this.config); + this.performance.image = Math.trunc(now() - timeStamp); + this.analyze('Get Image:'); + + // run segmentation preprocessing + if (this.config.segmentation.enabled && process && process.tensor) { + this.analyze('Start Segmentation:'); + this.state = 'run:segmentation'; + timeStamp = now(); + await segmentation.predict(process); + elapsedTime = Math.trunc(now() - timeStamp); + if (elapsedTime > 0) this.performance.segmentation = elapsedTime; + if (process.canvas) { + // replace input + process.tensor.dispose(); + process = image.process(process.canvas, this.config); + } + this.analyze('End Segmentation:'); + } + if (!process || !process.tensor) { log('could not convert input to tensor'); resolve({ error: 'could not convert input to tensor' }); return; } - this.performance.image = Math.trunc(now() - timeStamp); - this.analyze('Get Image:'); timeStamp = now(); this.config.skipFrame = await this.#skipFrame(process.tensor); @@ -497,7 +515,6 @@ export class Human { let bodyRes; let handRes; let objectRes; - let elapsedTime; // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion if (this.config.async) { @@ -573,19 +590,6 @@ export class Human { else if (this.performance.gesture) delete this.performance.gesture; } - // run segmentation - /* not triggered as part of detect - if (this.config.segmentation.enabled) { - this.analyze('Start Segmentation:'); - this.state = 'run:segmentation'; - timeStamp = now(); - await segmentation.predict(process, this.config); - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.segmentation = elapsedTime; - this.analyze('End Segmentation:'); - } - */ - this.performance.total = Math.trunc(now() - timeStart); this.state = 'idle'; this.result = { diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 01a148fa..54a2a21d 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -96,7 +96,7 @@ async function process(res, inputSize, outputShape, config) { // filter & sort results results = results - .filter((a, idx) => nmsIdx.includes(idx)) + .filter((_val, idx) => nmsIdx.includes(idx)) .sort((a, b) => (b.score - a.score)); return results; diff --git a/src/segmentation/segmentation.ts b/src/segmentation/segmentation.ts index e92a8ad4..ec392e96 100644 --- a/src/segmentation/segmentation.ts +++ b/src/segmentation/segmentation.ts @@ -7,13 +7,11 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as image from '../image/image'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; -// import * as blur from './blur'; type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; let model: GraphModel; let busy = false; -// let blurKernel; export async function load(config: Config): Promise { if (!model) { @@ -22,12 +20,13 @@ export async function load(config: Config): Promise { if (!model || !model['modelUrl']) log('load model failed:', config.segmentation.modelPath); else if (config.debug) log('load model:', model['modelUrl']); } else if (config.debug) log('cached model:', model['modelUrl']); - // if (!blurKernel) blurKernel = blur.getGaussianKernel(5, 1, 1); return model; } -export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }, config: Config): Promise { - if (!config.segmentation.enabled || !input.tensor || !input.canvas) return null; +export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }): Promise { + const width = input.tensor?.shape[1] || 0; + const height = input.tensor?.shape[2] || 0; + if (!input.tensor) return null; if (!model || !model.inputs[0].shape) return null; const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false); const norm = resizeInput.div(255); @@ -37,10 +36,6 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC tf.dispose(resizeInput); tf.dispose(norm); - const overlay = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); - overlay.width = input.canvas.width; - overlay.height = input.canvas.height; - const squeeze = tf.squeeze(res, 0); let resizeOutput; if (squeeze.shape[2] === 2) { @@ -53,7 +48,7 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC tf.dispose(bg); tf.dispose(fg); // running sofmax before unstack creates 2x2 matrix so we only take upper-left quadrant - const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [input.tensor?.shape[1], input.tensor?.shape[2]]); + const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]); // otherwise run softmax after unstack and use standard resize // resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]); resizeOutput = crop.squeeze(0); @@ -61,29 +56,34 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC tf.dispose(expand); tf.dispose(pad); } else { // model selfie has a single channel that we can use directly - resizeOutput = tf.image.resizeBilinear(squeeze, [input.tensor?.shape[1], input.tensor?.shape[2]]); + resizeOutput = tf.image.resizeBilinear(squeeze, [width, height]); } + if (typeof document === 'undefined') return resizeOutput.dataSync(); // we're running in nodejs so return alpha array as-is + + const overlay = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas'); + overlay.width = width; + overlay.height = height; if (tf.browser) await tf.browser.toPixels(resizeOutput, overlay); tf.dispose(resizeOutput); tf.dispose(squeeze); tf.dispose(res); // get alpha channel data - const alphaCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails - alphaCanvas.width = input.canvas.width; - alphaCanvas.height = input.canvas.height; + const alphaCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails + alphaCanvas.width = width; + alphaCanvas.height = height; const ctxAlpha = alphaCanvas.getContext('2d') as CanvasRenderingContext2D; ctxAlpha.filter = 'blur(8px'; await ctxAlpha.drawImage(overlay, 0, 0); - const alpha = ctxAlpha.getImageData(0, 0, input.canvas.width, input.canvas.height).data; + const alpha = ctxAlpha.getImageData(0, 0, width, height).data; // get original canvas merged with overlay - const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails - original.width = input.canvas.width; - original.height = input.canvas.height; + const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails + original.width = width; + original.height = height; const ctx = original.getContext('2d') as CanvasRenderingContext2D; - await ctx.drawImage(input.canvas, 0, 0); + if (input.canvas) await ctx.drawImage(input.canvas, 0, 0); // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation // best options are: darken, color-burn, multiply ctx.globalCompositeOperation = 'darken'; ctx.filter = 'blur(8px)'; // use css filter for bluring, can be done with gaussian blur manually instead @@ -99,10 +99,9 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC export async function process(input: Input, background: Input | undefined, config: Config): Promise { if (busy) return null; busy = true; - if (!config.segmentation.enabled) config.segmentation.enabled = true; // override config if (!model) await load(config); const img = image.process(input, config); - const alpha = await predict(img, config); + const alpha = await predict(img); tf.dispose(img.tensor); if (background && alpha) { diff --git a/test/test-node-gpu.js b/test/test-node-gpu.js index 9b5d83e5..eb3c8ed1 100644 --- a/test/test-node-gpu.js +++ b/test/test-node-gpu.js @@ -6,9 +6,6 @@ const config = { backend: 'tensorflow', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -20,6 +17,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: true }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/test/test-node-wasm.js b/test/test-node-wasm.js index d154da57..14bf3567 100644 --- a/test/test-node-wasm.js +++ b/test/test-node-wasm.js @@ -8,9 +8,6 @@ const config = { // wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -22,6 +19,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: false }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/test/test-node.js b/test/test-node.js index 169ac997..538445f1 100644 --- a/test/test-node.js +++ b/test/test-node.js @@ -6,9 +6,6 @@ const config = { backend: 'tensorflow', debug: false, async: false, - filter: { - enabled: true, - }, face: { enabled: true, detector: { enabled: true, rotation: true }, @@ -20,6 +17,8 @@ const config = { hand: { enabled: true }, body: { enabled: true }, object: { enabled: true }, + segmentation: { enabled: true }, + filter: { enabled: false }, }; test(Human, config); diff --git a/tsconfig.json b/tsconfig.json index 1e4fb9b8..9c034389 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,17 +7,16 @@ "typeRoots": ["node_modules/@types"], "outDir": "types", "declaration": true, + "allowSyntheticDefaultImports": true, "emitDeclarationOnly": true, "emitDecoratorMetadata": true, "experimentalDecorators": true, "importHelpers": true, - "noImplicitAny": false, "preserveConstEnums": true, "removeComments": false, "resolveJsonModule": true, "skipLibCheck": true, - "sourceMap": false, - "strictNullChecks": true, + "sourceMap": true, "allowJs": true, "baseUrl": "./", "paths": { @@ -25,10 +24,21 @@ "@tensorflow/tfjs-node/dist/io/file_system": ["node_modules/@tensorflow/tfjs-node/dist/io/file_system.js"], "@tensorflow/tfjs-core/dist/index": ["node_modules/@tensorflow/tfjs-core/dist/index.js"], "@tensorflow/tfjs-converter/dist/index": ["node_modules/@tensorflow/tfjs-converter/dist/index.js"] - } + }, + "strictNullChecks": true, + "noImplicitAny": false, + "noUnusedLocals": false, + "noImplicitReturns": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedParameters": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "allowUnreachableCode": false }, "formatCodeOptions": { "indentSize": 2, "tabSize": 2 }, "include": ["src/*", "src/***/*"], + "exclude": ["node_modules/"], "typedocOptions": { "excludePrivate": true, "excludeExternals": true, diff --git a/wiki b/wiki index c9408224..9e92e5ee 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c9408224d824368facc264c00e05d7b520d69051 +Subproject commit 9e92e5eec1e60b5ea58dbf1c4bbc67c828bcf673