enable body segmentation and background replacement

2021-06-05 11:54:49 -04:00 · 2021-06-05 11:54:49 -04:00 · 302cc31f59
parent 5c6ba688c9
commit 302cc31f59
9 changed files with 58 additions and 15 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -11,6 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**

 ### **HEAD -> main** 2021/06/04 mandic00@live.com

+- added experimental body segmentation module
 - add meet and selfie models
 - add live hints to demo
 - switch worker from module to iife importscripts
--- a/demo/index.html
+++ b/demo/index.html
@ -65,6 +65,7 @@
      .icon { width: 180px; text-align: -webkit-center; text-align: -moz-center; filter: grayscale(1); }
      .icon:hover { background: #505050; filter: grayscale(0); }
      .hint { opacity: 0; transition-duration: 0.5s; transition-property: opacity; font-style: italic; position: fixed; top: 5rem; padding: 8px; margin: 8px; box-shadow: 0 0 2px 2px #303030; }
+      .input-file { align-self: center; width: 5rem; }
    </style>
  </head>
  <body>
--- a/src/age/age.ts
+++ b/src/age/age.ts
@ -5,12 +5,16 @@

 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
+import { Config } from '../config';
+import { GraphModel, Tensor } from '../tfjs/types';
+
+let model: GraphModel;

-let model;
 let last = { age: 0 };
 let skipped = Number.MAX_SAFE_INTEGER;

-export async function load(config) {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function load(config: Config | any) {
  if (!model) {
    model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath));
    if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath);
@ -19,7 +23,8 @@ export async function load(config) {
  return model;
 }

-export async function predict(image, config) {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function predict(image: Tensor, config: Config | any) {
  if (!model) return null;
  if ((skipped < config.face.age.skipFrames) && config.skipFrame && last.age && (last.age > 0)) {
    skipped++;
--- a/src/config.ts
+++ b/src/config.ts
@ -353,6 +353,7 @@ const config: Config = {
  segmentation: {
    enabled: false,          // if segmentation is enabled, output result.canvas will be augmented
                             // with masked image containing only person output
+                             // segmentation is not triggered as part of detection and requires separate call to human.segmentation
    modelPath: 'selfie.json',  // experimental: object detection model, can be absolute path or relative to modelBasePath
                             // can be 'selfie' or 'meet'
  },
--- a/src/gender/gender.ts
+++ b/src/gender/gender.ts
@ -5,8 +5,10 @@

 import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
+import { Config } from '../config';
+import { GraphModel, Tensor } from '../tfjs/types';

-let model;
+let model: GraphModel;
 let last = { gender: '' };
 let skipped = Number.MAX_SAFE_INTEGER;
 let alternative = false;
@ -14,7 +16,8 @@ let alternative = false;
 // tuning values
 const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale

-export async function load(config) {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function load(config: Config | any) {
  if (!model) {
    model = await tf.loadGraphModel(join(config.modelBasePath, config.face.gender.modelPath));
    alternative = model.inputs[0].shape[3] === 1;
@ -24,7 +27,8 @@ export async function load(config) {
  return model;
 }

-export async function predict(image, config) {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function predict(image: Tensor, config: Config | any) {
  if (!model) return null;
  if ((skipped < config.face.gender.skipFrames) && config.skipFrame && last.gender !== '') {
    skipped++;
--- a/src/human.ts
+++ b/src/human.ts
@ -31,16 +31,16 @@ import { Tensor } from './tfjs/types';

 // export types
 export type { Config } from './config';
-export type { Result, Face, Hand, Body, Item, Gesture } from './result';
+export type { Result, Face, Hand, Body, Item, Gesture, Person } from './result';
 export type { DrawOptions } from './draw/draw';

 /** Defines all possible input types for **Human** detection
- * @typedef Input
+ * @typedef Input Type
 */
 export type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;

 /** Error message
- * @typedef Error
+ * @typedef Error Type
 */
 export type Error = { error: string };

@ -205,6 +205,7 @@ export class Human {

  /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings)
   * - Calculation is based on normalized Minkowski distance between
+   *
   * @param embedding1: face descriptor as array of numbers
   * @param embedding2: face descriptor as array of numbers
   * @returns similarity: number
@ -214,6 +215,19 @@ export class Human {
    return faceres.similarity(embedding1, embedding2);
  }

+  /**
+   * Segmentation method takes any input and returns processed canvas with body segmentation
+   * Optional parameter background is used to fill the background with specific input
+   * Segmentation is not triggered as part of detect process
+   *
+   * @param input: {@link Input}
+   * @param background?: {@link Input}
+   * @returns Canvas
+   */
+  segmentation(input: Input, background?: Input) {
+    return segmentation.process(input, background, this.config);
+  }
+
  /** Enhance method performs additional enhacements to face image previously detected for futher processing
   * @param input: Tensor as provided in human.result.face[n].tensor
   * @returns Tensor
@ -372,7 +386,8 @@ export class Human {
  /**
   * Runs interpolation using last known result and returns smoothened result
   * Interpolation is based on time since last known result so can be called independently
-   * @param result?: use specific result set to run interpolation on
+   *
+   * @param result?: {@link Result} optional use specific result set to run interpolation on
   * @returns result: {@link Result}
   */
  next = (result?: Result) => interpolate.calc(result || this.result) as Result;
@ -410,9 +425,10 @@ export class Human {
   * - Pre-process input: {@link Input}
   * - Run inference for all configured models
   * - Process and return result: {@link Result}
+   *
   * @param input: Input
-   * @param userConfig?: Config
-   * @returns result: Result
+   * @param userConfig?: {@link Config}
+   * @returns result: {@link Result}
  */
  async detect(input: Input, userConfig?: Config | Record<string, unknown>): Promise<Result | Error> {
    // detection happens inside a promise
@ -558,6 +574,7 @@ export class Human {
      }

      // run segmentation
+      /* not triggered as part of detect
      if (this.config.segmentation.enabled) {
        this.analyze('Start Segmentation:');
        this.state = 'run:segmentation';
@ -567,6 +584,7 @@ export class Human {
        if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
        this.analyze('End Segmentation:');
      }
+      */

      this.performance.total = Math.trunc(now() - timeStart);
      this.state = 'idle';
--- a/src/image/image.ts
+++ b/src/image/image.ts
@ -5,6 +5,9 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import * as fxImage from './imagefx';
 import { Tensor } from '../tfjs/types';
+import { Config } from '../config';
+
+type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;

 const maxSize = 2048;
 // internal temp canvases
@ -16,7 +19,7 @@ let fx;
 // process input image and return tensor
 // input can be tensor, imagedata, htmlimageelement, htmlvideoelement
 // input is resized and run through imagefx filter
-export function process(input, config): { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement } {
+export function process(input: Input, config: Config): { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement } {
  let tensor;
  if (!input) throw new Error('Human: Input is missing');
  // sanity checks since different browsers do not implement all dom elements
--- a/src/result.ts
+++ b/src/result.ts
@ -124,6 +124,7 @@ export interface Item {
 }

 /** Gesture results
+ * @typedef Gesture Type
 *
 * Array of individual results with one object per detected gesture
 * Each result has:
@ -137,6 +138,7 @@ export type Gesture =
  | { 'hand': number, gesture: string }

 /** Person getter
+* @interface Person Interface
 *
 * Each result has:
 * - id: person id
--- a/tsconfig.json
+++ b/tsconfig.json
@ -1,5 +1,6 @@
 {
  "compilerOptions": {
+    "noEmitOnError": false,
    "module": "es2020",
    "target": "es2018",
    "moduleResolution": "node",
@ -18,7 +19,14 @@
    "skipLibCheck": true,
    "sourceMap": false,
    "strictNullChecks": true,
-    "allowJs": true
+    "allowJs": true,
+    "baseUrl": "./",
+    "paths": {
+      "tslib": ["node_modules/tslib/tslib.d.ts"],
+      "@tensorflow/tfjs-node/dist/io/file_system": ["node_modules/@tensorflow/tfjs-node/dist/io/file_system.js"],
+      "@tensorflow/tfjs-core/dist/index": ["node_modules/@tensorflow/tfjs-core/dist/index.js"],
+      "@tensorflow/tfjs-converter/dist/index": ["node_modules/@tensorflow/tfjs-converter/dist/index.js"]
+    }
  },
  "formatCodeOptions": { "indentSize": 2, "tabSize": 2 },
  "include": ["src/*", "src/***/*"],
@ -35,6 +43,6 @@
    "entryPoints": "src/human.ts",
    "logLevel": "Info",
    "logger": "none",
-    "theme": "wiki/theme/",
+    "theme": "wiki/theme/"
  }
 }