diff --git a/CHANGELOG.md b/CHANGELOG.md index 645160fe..f2ed5c5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Repository: **** ### **HEAD -> main** 2021/06/04 mandic00@live.com +- added experimental body segmentation module - add meet and selfie models - add live hints to demo - switch worker from module to iife importscripts diff --git a/demo/index.html b/demo/index.html index 68ca4fcb..baaeb0bf 100644 --- a/demo/index.html +++ b/demo/index.html @@ -65,6 +65,7 @@ .icon { width: 180px; text-align: -webkit-center; text-align: -moz-center; filter: grayscale(1); } .icon:hover { background: #505050; filter: grayscale(0); } .hint { opacity: 0; transition-duration: 0.5s; transition-property: opacity; font-style: italic; position: fixed; top: 5rem; padding: 8px; margin: 8px; box-shadow: 0 0 2px 2px #303030; } + .input-file { align-self: center; width: 5rem; } diff --git a/src/age/age.ts b/src/age/age.ts index 58ada3cd..072c98ad 100644 --- a/src/age/age.ts +++ b/src/age/age.ts @@ -5,12 +5,16 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import { Config } from '../config'; +import { GraphModel, Tensor } from '../tfjs/types'; + +let model: GraphModel; -let model; let last = { age: 0 }; let skipped = Number.MAX_SAFE_INTEGER; -export async function load(config) { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function load(config: Config | any) { if (!model) { model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath)); if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath); @@ -19,7 +23,8 @@ export async function load(config) { return model; } -export async function predict(image, config) { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function predict(image: Tensor, config: Config | any) { if (!model) return null; if ((skipped < config.face.age.skipFrames) && config.skipFrame && last.age && (last.age > 0)) { skipped++; diff --git a/src/config.ts b/src/config.ts index d5a613ec..843d6540 100644 --- a/src/config.ts +++ b/src/config.ts @@ -353,6 +353,7 @@ const config: Config = { segmentation: { enabled: false, // if segmentation is enabled, output result.canvas will be augmented // with masked image containing only person output + // segmentation is not triggered as part of detection and requires separate call to human.segmentation modelPath: 'selfie.json', // experimental: object detection model, can be absolute path or relative to modelBasePath // can be 'selfie' or 'meet' }, diff --git a/src/gender/gender.ts b/src/gender/gender.ts index b918c4b2..ae192e69 100644 --- a/src/gender/gender.ts +++ b/src/gender/gender.ts @@ -5,8 +5,10 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import { Config } from '../config'; +import { GraphModel, Tensor } from '../tfjs/types'; -let model; +let model: GraphModel; let last = { gender: '' }; let skipped = Number.MAX_SAFE_INTEGER; let alternative = false; @@ -14,7 +16,8 @@ let alternative = false; // tuning values const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale -export async function load(config) { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function load(config: Config | any) { if (!model) { model = await tf.loadGraphModel(join(config.modelBasePath, config.face.gender.modelPath)); alternative = model.inputs[0].shape[3] === 1; @@ -24,7 +27,8 @@ export async function load(config) { return model; } -export async function predict(image, config) { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function predict(image: Tensor, config: Config | any) { if (!model) return null; if ((skipped < config.face.gender.skipFrames) && config.skipFrame && last.gender !== '') { skipped++; diff --git a/src/human.ts b/src/human.ts index 3c2533b5..f3fa2186 100644 --- a/src/human.ts +++ b/src/human.ts @@ -31,16 +31,16 @@ import { Tensor } from './tfjs/types'; // export types export type { Config } from './config'; -export type { Result, Face, Hand, Body, Item, Gesture } from './result'; +export type { Result, Face, Hand, Body, Item, Gesture, Person } from './result'; export type { DrawOptions } from './draw/draw'; /** Defines all possible input types for **Human** detection - * @typedef Input + * @typedef Input Type */ export type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; /** Error message - * @typedef Error + * @typedef Error Type */ export type Error = { error: string }; @@ -205,6 +205,7 @@ export class Human { /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings) * - Calculation is based on normalized Minkowski distance between + * * @param embedding1: face descriptor as array of numbers * @param embedding2: face descriptor as array of numbers * @returns similarity: number @@ -214,6 +215,19 @@ export class Human { return faceres.similarity(embedding1, embedding2); } + /** + * Segmentation method takes any input and returns processed canvas with body segmentation + * Optional parameter background is used to fill the background with specific input + * Segmentation is not triggered as part of detect process + * + * @param input: {@link Input} + * @param background?: {@link Input} + * @returns Canvas + */ + segmentation(input: Input, background?: Input) { + return segmentation.process(input, background, this.config); + } + /** Enhance method performs additional enhacements to face image previously detected for futher processing * @param input: Tensor as provided in human.result.face[n].tensor * @returns Tensor @@ -372,7 +386,8 @@ export class Human { /** * Runs interpolation using last known result and returns smoothened result * Interpolation is based on time since last known result so can be called independently - * @param result?: use specific result set to run interpolation on + * + * @param result?: {@link Result} optional use specific result set to run interpolation on * @returns result: {@link Result} */ next = (result?: Result) => interpolate.calc(result || this.result) as Result; @@ -410,9 +425,10 @@ export class Human { * - Pre-process input: {@link Input} * - Run inference for all configured models * - Process and return result: {@link Result} + * * @param input: Input - * @param userConfig?: Config - * @returns result: Result + * @param userConfig?: {@link Config} + * @returns result: {@link Result} */ async detect(input: Input, userConfig?: Config | Record): Promise { // detection happens inside a promise @@ -558,6 +574,7 @@ export class Human { } // run segmentation + /* not triggered as part of detect if (this.config.segmentation.enabled) { this.analyze('Start Segmentation:'); this.state = 'run:segmentation'; @@ -567,6 +584,7 @@ export class Human { if (elapsedTime > 0) this.performance.segmentation = elapsedTime; this.analyze('End Segmentation:'); } + */ this.performance.total = Math.trunc(now() - timeStart); this.state = 'idle'; diff --git a/src/image/image.ts b/src/image/image.ts index 653b6b62..4d92f59b 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -5,6 +5,9 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as fxImage from './imagefx'; import { Tensor } from '../tfjs/types'; +import { Config } from '../config'; + +type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; const maxSize = 2048; // internal temp canvases @@ -16,7 +19,7 @@ let fx; // process input image and return tensor // input can be tensor, imagedata, htmlimageelement, htmlvideoelement // input is resized and run through imagefx filter -export function process(input, config): { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement } { +export function process(input: Input, config: Config): { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement } { let tensor; if (!input) throw new Error('Human: Input is missing'); // sanity checks since different browsers do not implement all dom elements diff --git a/src/result.ts b/src/result.ts index d8933576..8dbf3084 100644 --- a/src/result.ts +++ b/src/result.ts @@ -124,6 +124,7 @@ export interface Item { } /** Gesture results + * @typedef Gesture Type * * Array of individual results with one object per detected gesture * Each result has: @@ -137,6 +138,7 @@ export type Gesture = | { 'hand': number, gesture: string } /** Person getter +* @interface Person Interface * * Each result has: * - id: person id diff --git a/src/segmentation/segmentation.ts b/src/segmentation/segmentation.ts index 126c07a0..f36b6085 100644 --- a/src/segmentation/segmentation.ts +++ b/src/segmentation/segmentation.ts @@ -4,15 +4,16 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import * as image from '../image/image'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; // import * as blur from './blur'; +type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; + let model: GraphModel; // let blurKernel; -export type Segmentation = boolean; - export async function load(config: Config): Promise { if (!model) { // @ts-ignore type mismatch on GraphModel @@ -24,9 +25,9 @@ export async function load(config: Config): Promise { return model; } -export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }, config: Config): Promise { - if (!config.segmentation.enabled || !input.tensor || !input.canvas) return false; - if (!model || !model.inputs[0].shape) return false; +export async function predict(input: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement }, config: Config): Promise { + if (!config.segmentation.enabled || !input.tensor || !input.canvas) return null; + if (!model || !model.inputs[0].shape) return null; const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false); const norm = resizeInput.div(255); const res = model.predict(norm) as Tensor; @@ -62,28 +63,69 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC resizeOutput = tf.image.resizeBilinear(squeeze, [input.tensor?.shape[1], input.tensor?.shape[2]]); } - // const blurred = blur.blur(resizeOutput, blurKernel); if (tf.browser) await tf.browser.toPixels(resizeOutput, overlay); - // tf.dispose(blurred); tf.dispose(resizeOutput); tf.dispose(squeeze); tf.dispose(res); + // get alpha channel data + const alphaCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails + alphaCanvas.width = input.canvas.width; + alphaCanvas.height = input.canvas.height; + const ctxAlpha = alphaCanvas.getContext('2d') as CanvasRenderingContext2D; + ctxAlpha.filter = 'blur(8px'; + await ctxAlpha.drawImage(overlay, 0, 0); + const alpha = ctxAlpha.getImageData(0, 0, input.canvas.width, input.canvas.height).data; + + // get original canvas merged with overlay const original = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(input.canvas.width, input.canvas.height) : document.createElement('canvas'); // need one more copy since input may already have gl context so 2d context fails original.width = input.canvas.width; original.height = input.canvas.height; const ctx = original.getContext('2d') as CanvasRenderingContext2D; - await ctx.drawImage(input.canvas, 0, 0); - // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation - // best options are: darken, color-burn, multiply + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/globalCompositeOperation // best options are: darken, color-burn, multiply ctx.globalCompositeOperation = 'darken'; ctx.filter = 'blur(8px)'; // use css filter for bluring, can be done with gaussian blur manually instead await ctx.drawImage(overlay, 0, 0); - ctx.globalCompositeOperation = 'source-in'; // reset + ctx.globalCompositeOperation = 'source-over'; // reset ctx.filter = 'none'; // reset input.canvas = original; - return true; + return alpha; +} + +export async function process(input: Input, background: Input | undefined, config: Config): Promise { + if (!config.segmentation.enabled) config.segmentation.enabled = true; // override config + if (!model) await load(config); + const img = image.process(input, config); + const alpha = await predict(img, config); + tf.dispose(img.tensor); + + if (background && alpha) { + const tmp = image.process(background, config); + const bg = tmp.canvas; + tf.dispose(tmp.tensor); + const fg = img.canvas; + const fgData = fg.getContext('2d')?.getImageData(0, 0, fg.width, fg.height).data as Uint8ClampedArray; + + const c = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(fg.width, fg.height) : document.createElement('canvas'); + c.width = fg.width; + c.height = fg.height; + const ctx = c.getContext('2d') as CanvasRenderingContext2D; + + ctx.globalCompositeOperation = 'copy'; // reset + ctx.drawImage(bg, 0, 0, c.width, c.height); + const cData = ctx.getImageData(0, 0, c.width, c.height) as ImageData; + for (let i = 0; i < c.width * c.height; i++) { // this should be done with globalCompositeOperation instead of looping through image data + cData.data[4 * i + 0] = ((255 - alpha[4 * i + 0]) / 255.0 * cData.data[4 * i + 0]) + (alpha[4 * i + 0] / 255.0 * fgData[4 * i + 0]); + cData.data[4 * i + 1] = ((255 - alpha[4 * i + 1]) / 255.0 * cData.data[4 * i + 1]) + (alpha[4 * i + 1] / 255.0 * fgData[4 * i + 1]); + cData.data[4 * i + 2] = ((255 - alpha[4 * i + 2]) / 255.0 * cData.data[4 * i + 2]) + (alpha[4 * i + 2] / 255.0 * fgData[4 * i + 2]); + cData.data[4 * i + 3] = ((255 - alpha[4 * i + 3]) / 255.0 * cData.data[4 * i + 3]) + (alpha[4 * i + 3] / 255.0 * fgData[4 * i + 3]); + } + ctx.putImageData(cData, 0, 0); + + return c; + } + return img.canvas; } diff --git a/tsconfig.json b/tsconfig.json index 6e9660bb..eb4813e6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "noEmitOnError": false, "module": "es2020", "target": "es2018", "moduleResolution": "node", @@ -18,7 +19,14 @@ "skipLibCheck": true, "sourceMap": false, "strictNullChecks": true, - "allowJs": true + "allowJs": true, + "baseUrl": "./", + "paths": { + "tslib": ["node_modules/tslib/tslib.d.ts"], + "@tensorflow/tfjs-node/dist/io/file_system": ["node_modules/@tensorflow/tfjs-node/dist/io/file_system.js"], + "@tensorflow/tfjs-core/dist/index": ["node_modules/@tensorflow/tfjs-core/dist/index.js"], + "@tensorflow/tfjs-converter/dist/index": ["node_modules/@tensorflow/tfjs-converter/dist/index.js"] + } }, "formatCodeOptions": { "indentSize": 2, "tabSize": 2 }, "include": ["src/*", "src/***/*"], @@ -35,6 +43,6 @@ "entryPoints": "src/human.ts", "logLevel": "Info", "logger": "none", - "theme": "wiki/theme/", + "theme": "wiki/theme/" } }