work on body segmentation

CHANGELOG.md

@@ -11,9 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/06/04 mandic00@live.com
 
-
+- add meet and selfie models
-### **update for tfjs 3.7.0** 2021/06/04 mandic00@live.com
-
 - add live hints to demo
 - switch worker from module to iife importscripts
 - release candidate

TODO.md

@@ -11,7 +11,7 @@ N/A
 ## In Progress
 
 - Switch to TypeScript 4.3
-- Implement segmentation model
+- Add backgrounds to segmentation
 
 ## Known Issues
 

demo/index.js

@@ -50,9 +50,7 @@ const userConfig = {
   hand: { enabled: false },
   body: { enabled: false },
   // body: { enabled: true, modelPath: 'posenet.json' },
-  // body: { enabled: true, modelPath: 'blazepose.json' },
+  segmentation: { enabled: true },
-  // segmentation: { enabled: true, modelPath: 'meet.json' },
-  // segmentation: { enabled: true, modelPath: 'selfie.json' },
   */
 };
 
@@ -211,8 +209,8 @@ async function drawResults(input) {
   // draw fps chart
   await menu.process.updateChart('FPS', ui.detectFPS);
 
-  // get updated canvas
+  // get updated canvas if missing or if we want buffering, but skip if segmentation is enabled
-  if (ui.buffered || !result.canvas) {
+  if (!result.canvas || (ui.buffered && !human.config.segmentation.enabled)) {
     const image = await human.image(input);
     result.canvas = image.canvas;
     human.tf.dispose(image.tensor);
@@ -489,6 +487,7 @@ async function processImage(input, title) {
     image.onload = async () => {
       if (ui.hintsThread) clearInterval(ui.hintsThread);
       ui.interpolated = false; // stop interpolating results if input is image
+      ui.buffered = false; // stop buffering result if input is image
       status(`processing image: ${title}`);
       const canvas = document.getElementById('canvas');
       image.width = image.naturalWidth;
@@ -676,6 +675,8 @@ function setupMenu() {
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('gestures', human.config.gesture, 'enabled', (val) => human.config.gesture.enabled = val);
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
+  menu.models.addBool('body segmentation', human.config.segmentation, 'enabled', (val) => human.config.segmentation.enabled = val);
+  menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('object detection', human.config.object, 'enabled', (val) => human.config.object.enabled = val);
   menu.models.addHTML('<hr style="border-style: inset; border-color: dimgray">');
   menu.models.addBool('face compare', compare, 'enabled', (val) => {

dist/human.node-gpu.js

@@ -10449,12 +10449,16 @@ async function predict11(input, config3) {
   if (squeeze4.shape[2] === 2) {
     const softmax = squeeze4.softmax();
     const [bg, fg] = tf20.unstack(softmax, 2);
-    tf20.dispose(softmax);
     const expand = fg.expandDims(2);
+    const pad = expand.expandDims(0);
+    tf20.dispose(softmax);
     tf20.dispose(bg);
     tf20.dispose(fg);
-    resizeOutput = tf20.image.resizeBilinear(expand, [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    const crop = tf20.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    resizeOutput = crop.squeeze(0);
+    tf20.dispose(crop);
     tf20.dispose(expand);
+    tf20.dispose(pad);
   } else {
     resizeOutput = tf20.image.resizeBilinear(squeeze4, [(_c = input.tensor) == null ? void 0 : _c.shape[1], (_d = input.tensor) == null ? void 0 : _d.shape[2]]);
   }
@@ -10463,10 +10467,17 @@ async function predict11(input, config3) {
   tf20.dispose(resizeOutput);
   tf20.dispose(squeeze4);
   tf20.dispose(res);
-  const ctx = input.canvas.getContext("2d");
+  const original = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement("canvas");
+  original.width = input.canvas.width;
+  original.height = input.canvas.height;
+  const ctx = original.getContext("2d");
+  await ctx.drawImage(input.canvas, 0, 0);
   ctx.globalCompositeOperation = "darken";
-  await (ctx == null ? void 0 : ctx.drawImage(overlay, 0, 0));
+  ctx.filter = "blur(8px)";
+  await ctx.drawImage(overlay, 0, 0);
   ctx.globalCompositeOperation = "source-in";
+  ctx.filter = "none";
+  input.canvas = original;
   return true;
 }
 

dist/human.node-wasm.js

@@ -10450,12 +10450,16 @@ async function predict11(input, config3) {
   if (squeeze4.shape[2] === 2) {
     const softmax = squeeze4.softmax();
     const [bg, fg] = tf20.unstack(softmax, 2);
-    tf20.dispose(softmax);
     const expand = fg.expandDims(2);
+    const pad = expand.expandDims(0);
+    tf20.dispose(softmax);
     tf20.dispose(bg);
     tf20.dispose(fg);
-    resizeOutput = tf20.image.resizeBilinear(expand, [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    const crop = tf20.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    resizeOutput = crop.squeeze(0);
+    tf20.dispose(crop);
     tf20.dispose(expand);
+    tf20.dispose(pad);
   } else {
     resizeOutput = tf20.image.resizeBilinear(squeeze4, [(_c = input.tensor) == null ? void 0 : _c.shape[1], (_d = input.tensor) == null ? void 0 : _d.shape[2]]);
   }
@@ -10464,10 +10468,17 @@ async function predict11(input, config3) {
   tf20.dispose(resizeOutput);
   tf20.dispose(squeeze4);
   tf20.dispose(res);
-  const ctx = input.canvas.getContext("2d");
+  const original = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement("canvas");
+  original.width = input.canvas.width;
+  original.height = input.canvas.height;
+  const ctx = original.getContext("2d");
+  await ctx.drawImage(input.canvas, 0, 0);
   ctx.globalCompositeOperation = "darken";
-  await (ctx == null ? void 0 : ctx.drawImage(overlay, 0, 0));
+  ctx.filter = "blur(8px)";
+  await ctx.drawImage(overlay, 0, 0);
   ctx.globalCompositeOperation = "source-in";
+  ctx.filter = "none";
+  input.canvas = original;
   return true;
 }
 

dist/human.node.js

@@ -10449,12 +10449,16 @@ async function predict11(input, config3) {
   if (squeeze4.shape[2] === 2) {
     const softmax = squeeze4.softmax();
     const [bg, fg] = tf20.unstack(softmax, 2);
-    tf20.dispose(softmax);
     const expand = fg.expandDims(2);
+    const pad = expand.expandDims(0);
+    tf20.dispose(softmax);
     tf20.dispose(bg);
     tf20.dispose(fg);
-    resizeOutput = tf20.image.resizeBilinear(expand, [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    const crop = tf20.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [(_a = input.tensor) == null ? void 0 : _a.shape[1], (_b = input.tensor) == null ? void 0 : _b.shape[2]]);
+    resizeOutput = crop.squeeze(0);
+    tf20.dispose(crop);
     tf20.dispose(expand);
+    tf20.dispose(pad);
   } else {
     resizeOutput = tf20.image.resizeBilinear(squeeze4, [(_c = input.tensor) == null ? void 0 : _c.shape[1], (_d = input.tensor) == null ? void 0 : _d.shape[2]]);
   }
@@ -10463,10 +10467,17 @@ async function predict11(input, config3) {
   tf20.dispose(resizeOutput);
   tf20.dispose(squeeze4);
   tf20.dispose(res);
-  const ctx = input.canvas.getContext("2d");
+  const original = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement("canvas");
+  original.width = input.canvas.width;
+  original.height = input.canvas.height;
+  const ctx = original.getContext("2d");
+  await ctx.drawImage(input.canvas, 0, 0);
   ctx.globalCompositeOperation = "darken";
-  await (ctx == null ? void 0 : ctx.drawImage(overlay, 0, 0));
+  ctx.filter = "blur(8px)";
+  await ctx.drawImage(overlay, 0, 0);
   ctx.globalCompositeOperation = "source-in";
+  ctx.filter = "none";
+  input.canvas = original;
   return true;
 }
 

server/build.log

@@ -1,21 +1,21 @@
-2021-06-04 13:51:15 INFO:  @vladmandic/human version 2.0.0 
+2021-06-04 20:20:35 INFO:  @vladmandic/human version 2.0.0 
-2021-06-04 13:51:15 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 
+2021-06-04 20:20:35 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 
-2021-06-04 13:51:15 INFO:  Toolchain: tfjs: 3.7.0 esbuild 0.12.6; typescript 4.2.4; typedoc: 0.20.36 eslint: 7.27.0 
+2021-06-04 20:20:35 INFO:  Toolchain: tfjs: 3.7.0 esbuild 0.12.6; typescript 4.2.4; typedoc: 0.20.36 eslint: 7.27.0 
-2021-06-04 13:51:15 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 
+2021-06-04 20:20:35 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 
-2021-06-04 13:51:15 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"} 
+2021-06-04 20:20:35 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"} 
-2021-06-04 13:51:15 STATE: Build for: node type: node: {"imports":41,"importBytes":424847,"outputBytes":373624,"outputFiles":"dist/human.node.js"} 
+2021-06-04 20:20:35 STATE: Build for: node type: node: {"imports":41,"importBytes":426297,"outputBytes":374093,"outputFiles":"dist/human.node.js"} 
-2021-06-04 13:51:15 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"} 
+2021-06-04 20:20:35 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"} 
-2021-06-04 13:51:15 STATE: Build for: nodeGPU type: node: {"imports":41,"importBytes":424855,"outputBytes":373628,"outputFiles":"dist/human.node-gpu.js"} 
+2021-06-04 20:20:35 STATE: Build for: nodeGPU type: node: {"imports":41,"importBytes":426305,"outputBytes":374097,"outputFiles":"dist/human.node-gpu.js"} 
-2021-06-04 13:51:15 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"} 
+2021-06-04 20:20:35 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"} 
-2021-06-04 13:51:15 STATE: Build for: nodeWASM type: node: {"imports":41,"importBytes":424922,"outputBytes":373700,"outputFiles":"dist/human.node-wasm.js"} 
+2021-06-04 20:20:35 STATE: Build for: nodeWASM type: node: {"imports":41,"importBytes":426372,"outputBytes":374169,"outputFiles":"dist/human.node-wasm.js"} 
-2021-06-04 13:51:15 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"} 
+2021-06-04 20:20:35 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"} 
-2021-06-04 13:51:15 STATE: Build for: browserNoBundle type: esm: {"imports":41,"importBytes":424949,"outputBytes":246353,"outputFiles":"dist/human.esm-nobundle.js"} 
+2021-06-04 20:20:35 STATE: Build for: browserNoBundle type: esm: {"imports":41,"importBytes":426399,"outputBytes":246661,"outputFiles":"dist/human.esm-nobundle.js"} 
-2021-06-04 13:51:16 STATE: Build for: browserBundle type: tfjs: {"modules":1299,"moduleBytes":4230827,"imports":7,"importBytes":2478,"outputBytes":1140320,"outputFiles":"dist/tfjs.esm.js"} 
+2021-06-04 20:20:36 STATE: Build for: browserBundle type: tfjs: {"modules":1299,"moduleBytes":4230827,"imports":7,"importBytes":2478,"outputBytes":1140320,"outputFiles":"dist/tfjs.esm.js"} 
-2021-06-04 13:51:16 STATE: Build for: browserBundle type: iife: {"imports":41,"importBytes":1563875,"outputBytes":1382597,"outputFiles":"dist/human.js"} 
+2021-06-04 20:20:36 STATE: Build for: browserBundle type: iife: {"imports":41,"importBytes":1565325,"outputBytes":1382891,"outputFiles":"dist/human.js"} 
-2021-06-04 13:51:17 STATE: Build for: browserBundle type: esm: {"imports":41,"importBytes":1563875,"outputBytes":1382589,"outputFiles":"dist/human.esm.js"} 
+2021-06-04 20:20:37 STATE: Build for: browserBundle type: esm: {"imports":41,"importBytes":1565325,"outputBytes":1382883,"outputFiles":"dist/human.esm.js"} 
-2021-06-04 13:51:17 INFO:  Running Linter: ["server/","demo/","src/","test/"] 
+2021-06-04 20:20:37 INFO:  Running Linter: ["server/","demo/","src/","test/"] 
-2021-06-04 13:51:45 INFO:  Linter complete: files: 69 errors: 0 warnings: 0 
+2021-06-04 20:21:06 INFO:  Linter complete: files: 69 errors: 0 warnings: 0 
-2021-06-04 13:51:45 INFO:  Generate types: ["src/human.ts"] 
+2021-06-04 20:21:06 INFO:  Generate types: ["src/human.ts"] 
-2021-06-04 13:51:50 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 
+2021-06-04 20:21:10 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 
-2021-06-04 13:51:50 INFO:  Generate TypeDocs: ["src/human.ts"] 
+2021-06-04 20:21:10 INFO:  Generate TypeDocs: ["src/human.ts"] 
-2021-06-04 13:52:08 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 
+2021-06-04 20:21:29 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 

src/config.ts

@@ -198,6 +198,8 @@ export interface Config {
   },
 
   /** Controlls and configures all body segmentation module
+   * if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output
+   *
    * - enabled: true/false
    * - modelPath: object detection model, can be absolute path or relative to modelBasePath
   */
@@ -349,7 +351,8 @@ const config: Config = {
   },
 
   segmentation: {
-    enabled: false,
+    enabled: false,          // if segmentation is enabled, output result.canvas will be augmented
+                             // with masked image containing only person output
     modelPath: 'selfie.json',  // experimental: object detection model, can be absolute path or relative to modelBasePath
                              // can be 'selfie' or 'meet'
   },

src/helpers.ts

@@ -39,3 +39,10 @@ export function mergeDeep(...objects) {
     return prev;
   }, {});
 }
+
+// helper function: return min and max from input array
+export const minmax = (data) => data.reduce((acc, val) => {
+  acc[0] = (acc[0] === undefined || val < acc[0]) ? val : acc[0];
+  acc[1] = (acc[1] === undefined || val > acc[1]) ? val : acc[1];
+  return acc;
+}, []);

src/segmentation/segmentation.ts

@@ -20,7 +20,7 @@ export async function load(config: Config): Promise<GraphModel> {
     if (!model || !model['modelUrl']) log('load model failed:', config.segmentation.modelPath);
     else if (config.debug) log('load model:', model['modelUrl']);
   } else if (config.debug) log('cached model:', model['modelUrl']);
-  // if (!blurKernel) blurKernel = blur.getGaussianKernel(50, 1, 1);
+  // if (!blurKernel) blurKernel = blur.getGaussianKernel(5, 1, 1);
   return model;
 }
 
@@ -30,6 +30,8 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
   const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false);
   const norm = resizeInput.div(255);
   const res = model.predict(norm) as Tensor;
+  // meet output:   1,256,256,1
+  // selfie output: 1,144,256,2
   tf.dispose(resizeInput);
   tf.dispose(norm);
 
@@ -39,16 +41,24 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
 
   const squeeze = tf.squeeze(res, 0);
   let resizeOutput;
-  if (squeeze.shape[2] === 2) { // model meet has two channels for fg and bg
+  if (squeeze.shape[2] === 2) {
+    // model meet has two channels for fg and bg
     const softmax = squeeze.softmax();
     const [bg, fg] = tf.unstack(softmax, 2);
-    tf.dispose(softmax);
     const expand = fg.expandDims(2);
+    const pad = expand.expandDims(0);
+    tf.dispose(softmax);
     tf.dispose(bg);
     tf.dispose(fg);
-    resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]);
+    // running sofmax before unstack creates 2x2 matrix so we only take upper-left quadrant
+    const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [input.tensor?.shape[1], input.tensor?.shape[2]]);
+    // otherwise run softmax after unstack and use standard resize
+    // resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]);
+    resizeOutput = crop.squeeze(0);
+    tf.dispose(crop);
     tf.dispose(expand);
-  } else { // model selfie has a single channel
+    tf.dispose(pad);
+  } else { // model selfie has a single channel that we can use directly
     resizeOutput = tf.image.resizeBilinear(squeeze, [input.tensor?.shape[1], input.tensor?.shape[2]]);
   }
 
@@ -59,17 +69,21 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
   tf.dispose(squeeze);
   tf.dispose(res);
 
-  const ctx = input.canvas.getContext('2d') as CanvasRenderingContext2D;
+  const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails
+  original.width = input.canvas.width;
+  original.height = input.canvas.height;
+  const ctx = original.getContext('2d') as CanvasRenderingContext2D;
+
+  await ctx.drawImage(input.canvas, 0, 0);
   // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation
   // best options are: darken, color-burn, multiply
   ctx.globalCompositeOperation = 'darken';
-  await ctx?.drawImage(overlay, 0, 0);
+  ctx.filter = 'blur(8px)'; // use css filter for bluring, can be done with gaussian blur manually instead
-  ctx.globalCompositeOperation = 'source-in';
+  await ctx.drawImage(overlay, 0, 0);
+  ctx.globalCompositeOperation = 'source-in'; // reset
+  ctx.filter = 'none'; // reset
+
+  input.canvas = original;
+
   return true;
 }
-
-/* Segmentation todo:
-- Smoothen
-- Get latest canvas in interpolate
-- Buffered fetches latest from video instead from interpolate
-*/

typedoc/interfaces/config.html

@@ -628,13 +628,14 @@
 					</aside>
 					<div class="tsd-comment tsd-typography">
 						<div class="lead">
-							<p>Controlls and configures all body segmentation module</p>
+							<p>Controlls and configures all body segmentation module
+							if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output</p>
+						</div>
 						<ul>
 							<li>enabled: true/false</li>
 							<li>modelPath: object detection model, can be absolute path or relative to modelBasePath</li>
 						</ul>
 					</div>
-					</div>
 					<div class="tsd-type-declaration">
 						<h4>Type declaration</h4>
 						<ul class="tsd-parameters">

types/config.d.ts

@@ -180,6 +180,8 @@ export interface Config {
         skipFrames: number;
     };
     /** Controlls and configures all body segmentation module
+     * if segmentation is enabled, output result.canvas will be augmented with masked image containing only person output
+     *
      * - enabled: true/false
      * - modelPath: object detection model, can be absolute path or relative to modelBasePath
     */

types/helpers.d.ts

@@ -5,3 +5,4 @@ export declare function join(folder: string, file: string): string;
 export declare function log(...msg: any[]): void;
 export declare const now: () => number;
 export declare function mergeDeep(...objects: any[]): any;
+export declare const minmax: (data: any) => any;

wiki

@@ -1 +1 @@
-Subproject commit 8e898a636f5254a3fe451b097c633c9965a8a680
+Subproject commit a69870f5763ae3fddd1243df10559aaf32c8f0da