modularize build platform

2021-06-05 17:51:46 -04:00 · 2021-06-05 17:51:46 -04:00 · 7f5e5aa00a
parent bf42a1c64e
commit 7f5e5aa00a
13 changed files with 79 additions and 67 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -11,9 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**

 ### **HEAD -> main** 2021/06/05 mandic00@live.com

-
-### **origin/main** 2021/06/05 mandic00@live.com
-
+- minor git corruption
 - unified build
 - enable body segmentation and background replacement
 - work on body segmentation
--- a/demo/facematch.js
+++ b/demo/facematch.js
@ -27,9 +27,8 @@ const userConfig = {
  hand: { enabled: false },
  gesture: { enabled: false },
  body: { enabled: false },
-  filter: {
-    enabled: false,
-  },
+  filter: { enabled: true },
+  segmentation: { enabled: false },
 };

 const human = new Human(userConfig); // new instance of human
--- a/demo/index.js
+++ b/demo/index.js
@ -31,6 +31,7 @@ let userConfig = {
  warmup: 'none',
  backend: 'humangl',
  wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/',
+  segmentation: { enabled: true },
  /*
  async: false,
  cacheSensitivity: 0,
@ -210,10 +211,9 @@ async function drawResults(input) {
  // draw fps chart
  await menu.process.updateChart('FPS', ui.detectFPS);

-  // get updated canvas if missing or if we want buffering, but skip if segmentation is enabled
-  if (userConfig.segmentation.enabled) {
+  if (userConfig.segmentation.enabled && ui.buffered) { // refresh segmentation if using buffered output
    result.canvas = await human.segmentation(input, ui.background, userConfig);
-  } else if (!result.canvas || ui.buffered) {
+  } else if (!result.canvas || ui.buffered) { // refresh with input if using buffered output or if missing canvas
    const image = await human.image(input);
    result.canvas = image.canvas;
    human.tf.dispose(image.tensor);
--- a/src/config.ts
+++ b/src/config.ts
@ -198,7 +198,10 @@ export interface Config {
  },

  /** Controlls and configures all body segmentation module
-   * if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output
+   * removes background from input containing person
+   * if segmentation is enabled it will run as preprocessing task before any other model
+   * alternatively leave it disabled and use it on-demand using human.segmentation method which can
+   * remove background or replace it with user-provided background
   *
   * - enabled: true/false
   * - modelPath: object detection model, can be absolute path or relative to modelBasePath
@ -351,9 +354,11 @@ const config: Config = {
  },

  segmentation: {
-    enabled: false,          // if segmentation is enabled, output result.canvas will be augmented
-                             // with masked image containing only person output
-                             // segmentation is not triggered as part of detection and requires separate call to human.segmentation
+    enabled: false,          // controlls and configures all body segmentation module
+                             // removes background from input containing person
+                             // if segmentation is enabled it will run as preprocessing task before any other model
+                             // alternatively leave it disabled and use it on-demand using human.segmentation method which can
+                             // remove background or replace it with user-provided background
    modelPath: 'selfie.json',  // experimental: object detection model, can be absolute path or relative to modelBasePath
                             // can be 'selfie' or 'meet'
  },
--- a/src/faceres/faceres.ts
+++ b/src/faceres/faceres.ts
@ -39,7 +39,7 @@ export function similarity(embedding1: Array<number>, embedding2: Array<number>,
  if (embedding1?.length !== embedding2?.length) return 0;
  // general minkowski distance, euclidean distance is limited case where order is 2
  const distance = 5.0 * embedding1
-    .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
+    .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
    .reduce((sum, now) => (sum + now), 0) // sum all distances
    ** (1 / order); // get root of
  const res = Math.max(0, 100 - distance) / 100.0;
--- a/src/human.ts
+++ b/src/human.ts
@ -435,6 +435,7 @@ export class Human {
    return new Promise(async (resolve) => {
      this.state = 'config';
      let timeStamp;
+      let elapsedTime;

      // update configuration
      this.config = mergeDeep(this.config, userConfig) as Config;
@ -473,14 +474,31 @@ export class Human {
      */

      timeStamp = now();
-      const process = image.process(input, this.config);
+      let process = image.process(input, this.config);
+      this.performance.image = Math.trunc(now() - timeStamp);
+      this.analyze('Get Image:');
+
+      // run segmentation preprocessing
+      if (this.config.segmentation.enabled && process && process.tensor) {
+        this.analyze('Start Segmentation:');
+        this.state = 'run:segmentation';
+        timeStamp = now();
+        await segmentation.predict(process);
+        elapsedTime = Math.trunc(now() - timeStamp);
+        if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
+        if (process.canvas) {
+          // replace input
+          process.tensor.dispose();
+          process = image.process(process.canvas, this.config);
+        }
+        this.analyze('End Segmentation:');
+      }
+
      if (!process || !process.tensor) {
        log('could not convert input to tensor');
        resolve({ error: 'could not convert input to tensor' });
        return;
      }
-      this.performance.image = Math.trunc(now() - timeStamp);
-      this.analyze('Get Image:');

      timeStamp = now();
      this.config.skipFrame = await this.#skipFrame(process.tensor);
@ -497,7 +515,6 @@ export class Human {
      let bodyRes;
      let handRes;
      let objectRes;
-      let elapsedTime;

      // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
      if (this.config.async) {
@ -573,19 +590,6 @@ export class Human {
        else if (this.performance.gesture) delete this.performance.gesture;
      }

-      // run segmentation
-      /* not triggered as part of detect
-      if (this.config.segmentation.enabled) {
-        this.analyze('Start Segmentation:');
-        this.state = 'run:segmentation';
-        timeStamp = now();
-        await segmentation.predict(process, this.config);
-        elapsedTime = Math.trunc(now() - timeStamp);
-        if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
-        this.analyze('End Segmentation:');
-      }
-      */
-
      this.performance.total = Math.trunc(now() - timeStart);
      this.state = 'idle';
      this.result = {
--- a/src/object/nanodet.ts
+++ b/src/object/nanodet.ts
@ -96,7 +96,7 @@ async function process(res, inputSize, outputShape, config) {

  // filter & sort results
  results = results
-    .filter((a, idx) => nmsIdx.includes(idx))
+    .filter((_val, idx) => nmsIdx.includes(idx))
    .sort((a, b) => (b.score - a.score));

  return results;
--- a/src/segmentation/segmentation.ts
+++ b/src/segmentation/segmentation.ts
@ -7,13 +7,11 @@ import * as tf from '../../dist/tfjs.esm.js';
 import * as image from '../image/image';
 import { GraphModel, Tensor } from '../tfjs/types';
 import { Config } from '../config';
-// import * as blur from './blur';

 type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;

 let model: GraphModel;
 let busy = false;
-// let blurKernel;

 export async function load(config: Config): Promise<GraphModel> {
  if (!model) {
@ -22,12 +20,13 @@ export async function load(config: Config): Promise<GraphModel> {
    if (!model || !model['modelUrl']) log('load model failed:', config.segmentation.modelPath);
    else if (config.debug) log('load model:', model['modelUrl']);
  } else if (config.debug) log('cached model:', model['modelUrl']);
-  // if (!blurKernel) blurKernel = blur.getGaussianKernel(5, 1, 1);
  return model;
 }

-export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }, config: Config): Promise<Uint8ClampedArray | null> {
-  if (!config.segmentation.enabled || !input.tensor || !input.canvas) return null;
+export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }): Promise<Uint8ClampedArray | null> {
+  const width = input.tensor?.shape[1] || 0;
+  const height = input.tensor?.shape[2] || 0;
+  if (!input.tensor) return null;
  if (!model || !model.inputs[0].shape) return null;
  const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false);
  const norm = resizeInput.div(255);
@ -37,10 +36,6 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
  tf.dispose(resizeInput);
  tf.dispose(norm);

-  const overlay = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas');
-  overlay.width = input.canvas.width;
-  overlay.height = input.canvas.height;
-
  const squeeze = tf.squeeze(res, 0);
  let resizeOutput;
  if (squeeze.shape[2] === 2) {
@ -53,7 +48,7 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
    tf.dispose(bg);
    tf.dispose(fg);
    // running sofmax before unstack creates 2x2 matrix so we only take upper-left quadrant
-    const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [input.tensor?.shape[1], input.tensor?.shape[2]]);
+    const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]);
    // otherwise run softmax after unstack and use standard resize
    // resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]);
    resizeOutput = crop.squeeze(0);
@ -61,29 +56,34 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
    tf.dispose(expand);
    tf.dispose(pad);
  } else { // model selfie has a single channel that we can use directly
-    resizeOutput = tf.image.resizeBilinear(squeeze, [input.tensor?.shape[1], input.tensor?.shape[2]]);
+    resizeOutput = tf.image.resizeBilinear(squeeze, [width, height]);
  }

+  if (typeof document === 'undefined') return resizeOutput.dataSync(); // we're running in nodejs so return alpha array as-is
+
+  const overlay = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas');
+  overlay.width = width;
+  overlay.height = height;
  if (tf.browser) await tf.browser.toPixels(resizeOutput, overlay);
  tf.dispose(resizeOutput);
  tf.dispose(squeeze);
  tf.dispose(res);

  // get alpha channel data
-  const alphaCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails
-  alphaCanvas.width = input.canvas.width;
-  alphaCanvas.height = input.canvas.height;
+  const alphaCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails
+  alphaCanvas.width = width;
+  alphaCanvas.height = height;
  const ctxAlpha = alphaCanvas.getContext('2d') as CanvasRenderingContext2D;
  ctxAlpha.filter = 'blur(8px';
  await ctxAlpha.drawImage(overlay, 0, 0);
-  const alpha = ctxAlpha.getImageData(0, 0, input.canvas.width, input.canvas.height).data;
+  const alpha = ctxAlpha.getImageData(0, 0, width, height).data;

  // get original canvas merged with overlay
-  const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails
-  original.width = input.canvas.width;
-  original.height = input.canvas.height;
+  const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails
+  original.width = width;
+  original.height = height;
  const ctx = original.getContext('2d') as CanvasRenderingContext2D;
-  await ctx.drawImage(input.canvas, 0, 0);
+  if (input.canvas) await ctx.drawImage(input.canvas, 0, 0);
  // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation // best options are: darken, color-burn, multiply
  ctx.globalCompositeOperation = 'darken';
  ctx.filter = 'blur(8px)'; // use css filter for bluring, can be done with gaussian blur manually instead
@ -99,10 +99,9 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
 export async function process(input: Input, background: Input | undefined, config: Config): Promise<HTMLCanvasElement | OffscreenCanvas | null> {
  if (busy) return null;
  busy = true;
-  if (!config.segmentation.enabled) config.segmentation.enabled = true; // override config
  if (!model) await load(config);
  const img = image.process(input, config);
-  const alpha = await predict(img, config);
+  const alpha = await predict(img);
  tf.dispose(img.tensor);

  if (background && alpha) {
--- a/test/test-node-gpu.js
+++ b/test/test-node-gpu.js
@ -6,9 +6,6 @@ const config = {
  backend: 'tensorflow',
  debug: false,
  async: false,
-  filter: {
-    enabled: true,
-  },
  face: {
    enabled: true,
    detector: { enabled: true, rotation: true },
@ -20,6 +17,8 @@ const config = {
  hand: { enabled: true },
  body: { enabled: true },
  object: { enabled: true },
+  segmentation: { enabled: true },
+  filter: { enabled: false },
 };

 test(Human, config);
--- a/test/test-node-wasm.js
+++ b/test/test-node-wasm.js
@ -8,9 +8,6 @@ const config = {
  // wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/',
  debug: false,
  async: false,
-  filter: {
-    enabled: true,
-  },
  face: {
    enabled: true,
    detector: { enabled: true, rotation: true },
@ -22,6 +19,8 @@ const config = {
  hand: { enabled: true },
  body: { enabled: true },
  object: { enabled: false },
+  segmentation: { enabled: true },
+  filter: { enabled: false },
 };

 test(Human, config);
--- a/test/test-node.js
+++ b/test/test-node.js
@ -6,9 +6,6 @@ const config = {
  backend: 'tensorflow',
  debug: false,
  async: false,
-  filter: {
-    enabled: true,
-  },
  face: {
    enabled: true,
    detector: { enabled: true, rotation: true },
@ -20,6 +17,8 @@ const config = {
  hand: { enabled: true },
  body: { enabled: true },
  object: { enabled: true },
+  segmentation: { enabled: true },
+  filter: { enabled: false },
 };

 test(Human, config);
--- a/tsconfig.json
+++ b/tsconfig.json
@ -7,17 +7,16 @@
    "typeRoots": ["node_modules/@types"],
    "outDir": "types",
    "declaration": true,
+    "allowSyntheticDefaultImports": true,
    "emitDeclarationOnly": true,
    "emitDecoratorMetadata": true,
    "experimentalDecorators": true,
    "importHelpers": true,
-    "noImplicitAny": false,
    "preserveConstEnums": true,
    "removeComments": false,
    "resolveJsonModule": true,
    "skipLibCheck": true,
-    "sourceMap": false,
-    "strictNullChecks": true,
+    "sourceMap": true,
    "allowJs": true,
    "baseUrl": "./",
    "paths": {
@ -25,10 +24,21 @@
      "@tensorflow/tfjs-node/dist/io/file_system": ["node_modules/@tensorflow/tfjs-node/dist/io/file_system.js"],
      "@tensorflow/tfjs-core/dist/index": ["node_modules/@tensorflow/tfjs-core/dist/index.js"],
      "@tensorflow/tfjs-converter/dist/index": ["node_modules/@tensorflow/tfjs-converter/dist/index.js"]
-    }
+    },
+    "strictNullChecks": true,
+    "noImplicitAny": false,
+    "noUnusedLocals": false,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+    "alwaysStrict": true,
+    "noUnusedParameters": true,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "allowUnreachableCode": false
  },
  "formatCodeOptions": { "indentSize": 2, "tabSize": 2 },
  "include": ["src/*", "src/***/*"],
+  "exclude": ["node_modules/"],
  "typedocOptions": {
    "excludePrivate": true,
    "excludeExternals": true,
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit c9408224d824368facc264c00e05d7b520d69051
+Subproject commit 9e92e5eec1e60b5ea58dbf1c4bbc67c828bcf673