diff --git a/build.json b/build.json index 6d7eb814..6b0bb75a 100644 --- a/build.json +++ b/build.json @@ -13,7 +13,7 @@ "locations": ["dist/*", "types/*", "typedoc/*"] }, "lint": { - "locations": [ "src/**/*.ts", "test/*.js", "demo/**/*.js" ], + "locations": [ "*.json", "src/**/*.ts", "test/**/*.js", "demo/**/*.js" ], "rules": { } }, "changelog": { @@ -133,7 +133,7 @@ ] }, "watch": { - "locations": [ "src/**", "tfjs/*" ] + "locations": [ "src/**/*", "tfjs/**/*" ] }, "typescript": { "allowJs": false diff --git a/src/age/age.ts b/src/age/age.ts index b3c98462..7aa03f7a 100644 --- a/src/age/age.ts +++ b/src/age/age.ts @@ -5,8 +5,8 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { Config } from '../config'; -import { GraphModel, Tensor } from '../tfjs/types'; +import type { Config } from '../config'; +import type { GraphModel, Tensor } from '../tfjs/types'; let model: GraphModel; diff --git a/src/config.ts b/src/config.ts index 41fb7c67..363782a4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -186,8 +186,8 @@ export interface GestureConfig { */ export interface Config { /** Backend used for TFJS operations */ - // backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu' | null, - backend: string; + backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu', + // backend: string; /** Path to *.wasm files if backend is set to `wasm` */ wasmPath: string, @@ -202,8 +202,8 @@ export interface Config { * - warmup pre-initializes all models for faster inference but can take significant time on startup * - only used for `webgl` and `humangl` backends */ - // warmup: 'none' | 'face' | 'full' | 'body' | string, - warmup: string; + warmup: 'none' | 'face' | 'full' | 'body', + // warmup: string; /** Base model path (typically starting with file://, http:// or https://) for all models * - individual modelPath values are relative to this path diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index 28d29910..eef80a28 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -4,9 +4,9 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { BodyResult } from '../result'; -import { GraphModel, Tensor } from '../tfjs/types'; -import { Config } from '../config'; +import type { BodyResult } from '../result'; +import type { GraphModel, Tensor } from '../tfjs/types'; +import type { Config } from '../config'; let model: GraphModel; diff --git a/src/emotion/emotion.ts b/src/emotion/emotion.ts index 1fe56e73..de2caf54 100644 --- a/src/emotion/emotion.ts +++ b/src/emotion/emotion.ts @@ -3,8 +3,8 @@ */ import { log, join } from '../helpers'; -import { Config } from '../config'; -import { Tensor, GraphModel } from '../tfjs/types'; +import type { Config } from '../config'; +import type { Tensor, GraphModel } from '../tfjs/types'; import * as tf from '../../dist/tfjs.esm.js'; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; diff --git a/src/face.ts b/src/face.ts index 9ec100b7..94eeb46d 100644 --- a/src/face.ts +++ b/src/face.ts @@ -8,8 +8,8 @@ import * as tf from '../dist/tfjs.esm.js'; import * as facemesh from './blazeface/facemesh'; import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; -import { FaceResult } from './result'; -import { Tensor } from './tfjs/types'; +import type { FaceResult } from './result'; +import type { Tensor } from './tfjs/types'; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const rad2deg = (theta) => Math.round((theta * 180) / Math.PI); @@ -250,7 +250,6 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): rotation, tensor, }); - parent.analyze('End Face'); } parent.analyze('End FaceMesh:'); diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 9bbd773f..75f69901 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -6,8 +6,8 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { Tensor, GraphModel } from '../tfjs/types'; -import { Config } from '../config'; +import type { Tensor, GraphModel } from '../tfjs/types'; +import type { Config } from '../config'; let model: GraphModel; const last: Array<{ @@ -140,7 +140,8 @@ export async function predict(image: Tensor, config: Config, idx, count) { } const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1); const age = (await argmax.data())[0]; - const all = await resT.find((t) => t.shape[1] === 100).data(); // inside tf.tidy + tf.dispose(argmax); + const all = await resT.find((t) => t.shape[1] === 100).data(); obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; const desc = resT.find((t) => t.shape[1] === 1024); @@ -151,7 +152,6 @@ export async function predict(image: Tensor, config: Config, idx, count) { obj.descriptor = [...descriptor]; resT.forEach((t) => tf.dispose(t)); } - last[idx] = obj; lastCount = count; resolve(obj); diff --git a/src/gender/gender.ts b/src/gender/gender.ts index bd294ca1..bbabe89c 100644 --- a/src/gender/gender.ts +++ b/src/gender/gender.ts @@ -5,8 +5,8 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { Config } from '../config'; -import { GraphModel, Tensor } from '../tfjs/types'; +import type { Config } from '../config'; +import type { GraphModel, Tensor } from '../tfjs/types'; let model: GraphModel; let last = { gender: '' }; diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index e4d727d6..51489961 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -2,7 +2,7 @@ * Gesture detection module */ -import { GestureResult } from '../result'; +import type { GestureResult } from '../result'; import * as fingerPose from '../fingerpose/fingerpose'; /** diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index 762a25b1..63ccf353 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -1,7 +1,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as anchors from './anchors'; -import { Tensor, GraphModel } from '../tfjs/types'; +import type { Tensor, GraphModel } from '../tfjs/types'; export class HandDetector { model: GraphModel; diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index 4db952b4..9e8cdf0a 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -1,8 +1,8 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as util from './util'; -import * as detector from './handdetector'; -import { Tensor, GraphModel } from '../tfjs/types'; +import type * as detector from './handdetector'; +import type { Tensor, GraphModel } from '../tfjs/types'; import { env } from '../env'; const palmBoxEnlargeFactor = 5; // default 3 diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 0ecd6f1b..212934a3 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -7,9 +7,9 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; import * as handpipeline from './handpipeline'; import * as fingerPose from '../fingerpose/fingerpose'; -import { HandResult } from '../result'; -import { Tensor, GraphModel } from '../tfjs/types'; -import { Config } from '../config'; +import type { HandResult } from '../result'; +import type { Tensor, GraphModel } from '../tfjs/types'; +import type { Config } from '../config'; const meshAnnotations = { thumb: [1, 2, 3, 4], diff --git a/src/human.ts b/src/human.ts index eedf4342..e0f3ed13 100644 --- a/src/human.ts +++ b/src/human.ts @@ -4,7 +4,7 @@ import { log, now, mergeDeep } from './helpers'; import { Config, defaults } from './config'; -import { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult } from './result'; +import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult } from './result'; import * as tf from '../dist/tfjs.esm.js'; import * as models from './models'; import * as face from './face'; @@ -27,7 +27,7 @@ import * as env from './env'; import * as backend from './tfjs/backend'; import * as app from '../package.json'; import * as warmups from './warmup'; -import { Tensor, GraphModel } from './tfjs/types'; +import type { Tensor, GraphModel } from './tfjs/types'; // export types export * from './config'; @@ -38,7 +38,7 @@ export { env } from './env'; /** Defines all possible input types for **Human** detection * @typedef Input Type */ -export type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; +export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.env.Canvas; /** Events dispatched by `human.events` * - `create`: triggered when Human object is instantiated @@ -257,7 +257,7 @@ export class Human { * @returns Canvas */ segmentation(input: Input, background?: Input) { - return segmentation.process(input, background, this.config); + return input ? segmentation.process(input, background, this.config) : null; } /** Enhance method performs additional enhacements to face image previously detected for futher this.processing @@ -373,28 +373,28 @@ export class Human { await this.load(); timeStamp = now(); - this.process = image.process(input, this.config); - const inputTensor = this.process.tensor; + let img = image.process(input, this.config); + this.process = img; this.performance.image = Math.trunc(now() - timeStamp); this.analyze('Get Image:'); // run segmentation prethis.processing - if (this.config.segmentation.enabled && this.process && inputTensor) { + if (this.config.segmentation.enabled && this.process && img.tensor && img.canvas) { this.analyze('Start Segmentation:'); this.state = 'run:segmentation'; timeStamp = now(); - await segmentation.predict(this.process); + await segmentation.predict(img); elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.segmentation = elapsedTime; - if (this.process.canvas) { + if (img.canvas) { // replace input - tf.dispose(inputTensor); - this.process = image.process(this.process.canvas, this.config); + tf.dispose(img.tensor); + img = image.process(img.canvas, this.config); } this.analyze('End Segmentation:'); } - if (!this.process || !inputTensor) { + if (!img.tensor) { log('could not convert input to tensor'); resolve({ error: 'could not convert input to tensor' }); return; @@ -402,7 +402,7 @@ export class Human { this.emit('image'); timeStamp = now(); - this.config.skipFrame = await image.skip(this, inputTensor); + this.config.skipFrame = await image.skip(this.config, img.tensor); if (!this.performance.frames) this.performance.frames = 0; if (!this.performance.cached) this.performance.cached = 0; (this.performance.frames as number)++; @@ -419,12 +419,12 @@ export class Human { // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion if (this.config.async) { - faceRes = this.config.face.enabled ? face.detectFace(this, inputTensor) : []; + faceRes = this.config.face.enabled ? face.detectFace(this, img.tensor) : []; if (this.performance.face) delete this.performance.face; } else { this.state = 'run:face'; timeStamp = now(); - faceRes = this.config.face.enabled ? await face.detectFace(this, inputTensor) : []; + faceRes = this.config.face.enabled ? await face.detectFace(this, img.tensor) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.face = elapsedTime; } @@ -432,18 +432,18 @@ export class Human { // run body: can be posenet, blazepose, efficientpose, movenet this.analyze('Start Body:'); if (this.config.async) { - if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(inputTensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(img.tensor, this.config) : []; if (this.performance.body) delete this.performance.body; } else { this.state = 'run:body'; timeStamp = now(); - if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(inputTensor, this.config) : []; - else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(inputTensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(img.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(img.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.body = elapsedTime; } @@ -452,12 +452,12 @@ export class Human { // run handpose this.analyze('Start Hand:'); if (this.config.async) { - handRes = this.config.hand.enabled ? handpose.predict(inputTensor, this.config) : []; + handRes = this.config.hand.enabled ? handpose.predict(img.tensor, this.config) : []; if (this.performance.hand) delete this.performance.hand; } else { this.state = 'run:hand'; timeStamp = now(); - handRes = this.config.hand.enabled ? await handpose.predict(inputTensor, this.config) : []; + handRes = this.config.hand.enabled ? await handpose.predict(img.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.hand = elapsedTime; } @@ -466,14 +466,14 @@ export class Human { // run nanodet this.analyze('Start Object:'); if (this.config.async) { - if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(inputTensor, this.config) : []; - else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(inputTensor, this.config) : []; + if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(img.tensor, this.config) : []; + else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(img.tensor, this.config) : []; if (this.performance.object) delete this.performance.object; } else { this.state = 'run:object'; timeStamp = now(); - if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(inputTensor, this.config) : []; - else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(inputTensor, this.config) : []; + if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(img.tensor, this.config) : []; + else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(img.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.object = elapsedTime; } @@ -507,7 +507,7 @@ export class Human { }; // finally dispose input tensor - tf.dispose(inputTensor); + tf.dispose(img.tensor); // log('Result:', result); this.emit('detect'); diff --git a/src/image/image.ts b/src/image/image.ts index 74664e70..3bf26a07 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -4,11 +4,11 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as fxImage from './imagefx'; -import { Tensor } from '../tfjs/types'; -import { Config } from '../config'; +import type { Tensor } from '../tfjs/types'; +import type { Config } from '../config'; import { env } from '../env'; -type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; +type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas; const maxSize = 2048; // internal temp canvases @@ -17,6 +17,25 @@ let outCanvas; // @ts-ignore // imagefx is js module that should be converted to a class let fx: fxImage.GLImageFilter | null; // instance of imagefx +export function canvas(width, height) { + let c; + if (env.browser) { + if (typeof OffscreenCanvas !== 'undefined') { + c = new OffscreenCanvas(width, height); + } else { + c = document.createElement('canvas'); + c.width = width; + c.height = height; + } + } else { + // @ts-ignore // env.canvas is an external monkey-patch + // eslint-disable-next-line new-cap + c = (typeof env.Canvas !== 'undefined') ? new env.Canvas(width, height) : null; + } + if (!c) throw new Error('Human: Cannot create canvas'); + return c; +} + // process input image and return tensor // input can be tensor, imagedata, htmlimageelement, htmlvideoelement // input is resized and run through imagefx filter @@ -27,6 +46,7 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, if ( !(input instanceof tf.Tensor) && !(typeof Image !== 'undefined' && input instanceof Image) + && !(typeof env.Canvas !== 'undefined' && input instanceof env.Canvas) && !(typeof ImageData !== 'undefined' && input instanceof ImageData) && !(typeof ImageBitmap !== 'undefined' && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== 'undefined' && input instanceof HTMLImageElement) @@ -39,8 +59,8 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, } if (input instanceof tf.Tensor) { // if input is tensor, use as-is - if ((input as Tensor).shape && (input as Tensor).shape.length === 4 && (input as Tensor).shape[0] === 1 && (input as Tensor).shape[3] === 3) tensor = tf.clone(input); - else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${(input as Tensor).shape}`); + if ((input as unknown as Tensor).shape && (input as unknown as Tensor).shape.length === 4 && (input as unknown as Tensor).shape[0] === 1 && (input as unknown as Tensor).shape[3] === 3) tensor = tf.clone(input); + else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${(input as unknown as Tensor).shape}`); } else { // check if resizing will be needed const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0)); @@ -63,15 +83,11 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, if ((config.filter.height || 0) > 0) targetHeight = config.filter.height; else if ((config.filter.width || 0) > 0) targetHeight = originalHeight * ((config.filter.width || 0) / originalWidth); if (!targetWidth || !targetHeight) throw new Error('Human: Input cannot determine dimension'); - if (!inCanvas || (inCanvas?.width !== targetWidth) || (inCanvas?.height !== targetHeight)) { - inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); - if (inCanvas?.width !== targetWidth) inCanvas.width = targetWidth; - if (inCanvas?.height !== targetHeight) inCanvas.height = targetHeight; - } + if (!inCanvas || (inCanvas?.width !== targetWidth) || (inCanvas?.height !== targetHeight)) inCanvas = canvas(targetWidth, targetHeight); // draw input to our canvas const ctx = inCanvas.getContext('2d'); - if (input instanceof ImageData) { + if ((typeof ImageData !== 'undefined') && (input instanceof ImageData)) { ctx.putImageData(input, 0, 0); } else { if (config.filter.flip && typeof ctx.translate !== 'undefined') { @@ -83,11 +99,10 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, inCanvas?.width, inCanvas?.height); } } - // imagefx transforms using gl if (config.filter.enabled) { if (!fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas?.height !== outCanvas?.height)) { - outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(inCanvas?.width, inCanvas?.height) : document.createElement('canvas'); + outCanvas = canvas(inCanvas?.width, inCanvas?.height); if (outCanvas?.width !== inCanvas?.width) outCanvas.width = inCanvas?.width; if (outCanvas?.height !== inCanvas?.height) outCanvas.height = inCanvas?.height; // log('created FX filter'); @@ -146,45 +161,58 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, if (outCanvas.data) { // if we have data, just convert to tensor const shape = [outCanvas.height, outCanvas.width, 3]; pixels = tf.tensor3d(outCanvas.data, shape, 'int32'); - } else if (outCanvas instanceof ImageData) { // if input is imagedata, just use it + } else if ((typeof ImageData !== 'undefined') && (outCanvas instanceof ImageData)) { // if input is imagedata, just use it pixels = tf.browser ? tf.browser.fromPixels(outCanvas) : null; } else if (config.backend === 'webgl' || config.backend === 'humangl') { // tf kernel-optimized method to get imagedata // we cant use canvas as-is as it already has a context, so we do a silly one more canvas - const tempCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); + const tempCanvas = canvas(targetWidth, targetHeight); tempCanvas.width = targetWidth; tempCanvas.height = targetHeight; const tempCtx = tempCanvas.getContext('2d'); tempCtx?.drawImage(outCanvas, 0, 0); - pixels = tf.browser ? tf.browser.fromPixels(tempCanvas) : null; + pixels = (tf.browser && env.browser) ? tf.browser.fromPixels(tempCanvas) : null; } else { // cpu and wasm kernel does not implement efficient fromPixels method // we cant use canvas as-is as it already has a context, so we do a silly one more canvas and do fromPixels on ImageData instead - const tempCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); + const tempCanvas = canvas(targetWidth, targetHeight); tempCanvas.width = targetWidth; tempCanvas.height = targetHeight; const tempCtx = tempCanvas.getContext('2d'); - tempCtx?.drawImage(outCanvas, 0, 0); - const data = tempCtx?.getImageData(0, 0, targetWidth, targetHeight); - pixels = tf.browser ? tf.browser.fromPixels(data) : null; + tempCtx.drawImage(outCanvas, 0, 0); + const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight); + if (tf.browser && env.browser) { + pixels = tf.browser.fromPixels(data); + } else { + pixels = tf.tidy(() => { + const imageData = tf.tensor(Array.from(data.data), [targetWidth, targetHeight, 4]); + const channels = tf.split(imageData, 4, 2); // split rgba to channels + const rgb = tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha + const expand = tf.reshape(rgb, [imageData.shape[0], imageData.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead + return expand; + }); + } } if (pixels) { const casted = tf.cast(pixels, 'float32'); tensor = tf.expandDims(casted, 0); tf.dispose(pixels); tf.dispose(casted); + } else { + tensor = tf.zeros([1, targetWidth, targetHeight, 3]); + throw new Error('Human: Cannot create tensor from input'); } } } - const canvas = config.filter.return ? outCanvas : null; - return { tensor, canvas }; + return { tensor, canvas: (config.filter.return ? outCanvas : null) }; } let lastInputSum = 0; let lastCacheDiff = 1; -export async function skip(instance, input: Tensor) { - if (instance.config.cacheSensitivity === 0) return false; +export async function skip(config, input: Tensor) { + if (config.cacheSensitivity === 0) return false; const resizeFact = 32; if (!input.shape[1] || !input.shape[2]) return false; const reduced: Tensor = tf.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); + // use tensor sum /* const sumT = this.tf.sum(reduced); @@ -193,17 +221,17 @@ export async function skip(instance, input: Tensor) { */ // use js loop sum, faster than uploading tensor to gpu calculating and downloading back const reducedData = await reduced.data(); // raw image rgb array + tf.dispose(reduced); let sum = 0; for (let i = 0; i < reducedData.length / 3; i++) sum += reducedData[3 * i + 2]; // look only at green value of each pixel - reduced.dispose(); const diff = 100 * (Math.max(sum, lastInputSum) / Math.min(sum, lastInputSum) - 1); lastInputSum = sum; // if previous frame was skipped, skip this frame if changed more than cacheSensitivity // if previous frame was not skipped, then look for cacheSensitivity or difference larger than one in previous frame to avoid resetting cache in subsequent frames unnecessarily - const skipFrame = diff < Math.max(instance.config.cacheSensitivity, lastCacheDiff); + const skipFrame = diff < Math.max(config.cacheSensitivity, lastCacheDiff); // if difference is above 10x threshold, don't use last value to force reset cache for significant change of scenes or images - lastCacheDiff = diff > 10 * instance.config.cacheSensitivity ? 0 : diff; + lastCacheDiff = diff > 10 * config.cacheSensitivity ? 0 : diff; // console.log('skipFrame', skipFrame, this.config.cacheSensitivity, diff); return skipFrame; } diff --git a/src/models.ts b/src/models.ts index 3f1527cf..c419bdec 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1,5 +1,5 @@ import { log } from './helpers'; -import { GraphModel } from './tfjs/types'; +import type { GraphModel } from './tfjs/types'; import * as facemesh from './blazeface/facemesh'; import * as faceres from './faceres/faceres'; import * as emotion from './emotion/emotion'; diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index d38b5f3c..fd4c9183 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -4,9 +4,9 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { BodyResult } from '../result'; -import { GraphModel, Tensor } from '../tfjs/types'; -import { Config } from '../config'; +import type { BodyResult } from '../result'; +import type { GraphModel, Tensor } from '../tfjs/types'; +import type { Config } from '../config'; let model: GraphModel; diff --git a/src/object/centernet.ts b/src/object/centernet.ts index d86dd7cd..ebbaad97 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -5,9 +5,9 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; -import { ObjectResult } from '../result'; -import { GraphModel, Tensor } from '../tfjs/types'; -import { Config } from '../config'; +import type { ObjectResult } from '../result'; +import type { GraphModel, Tensor } from '../tfjs/types'; +import type { Config } from '../config'; import { env } from '../env'; let model; @@ -36,6 +36,7 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) { tf.dispose(squeezeT); const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x const boxesT = tf.squeeze(stackT); + tf.dispose(stackT); const scoresT = tf.squeeze(arr[4]); const classesT = tf.squeeze(arr[5]); arr.forEach((t) => tf.dispose(t)); @@ -86,6 +87,7 @@ export async function predict(input: Tensor, config: Config): Promise, bodies: Array, hands: Array, gestures: Array, shape: Array | undefined): Array { let id = 0; diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index 697f62a9..cc34153d 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -1,5 +1,5 @@ import * as kpt from './keypoints'; -import { BodyResult } from '../result'; +import type { BodyResult } from '../result'; export function eitherPointDoesntMeetConfidence(a: number, b: number, minConfidence: number) { return (a < minConfidence || b < minConfidence); diff --git a/src/result.ts b/src/result.ts index 66f7027b..1fc967d9 100644 --- a/src/result.ts +++ b/src/result.ts @@ -2,8 +2,8 @@ * Type definitions for Human result object */ -import { Tensor } from './tfjs/types'; -import { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; +import type { Tensor } from './tfjs/types'; +import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; /** Face results * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models @@ -186,7 +186,7 @@ export interface Result { /** global performance object with timing values for each operation */ performance: Record, /** optional processed canvas that can be used to draw input on screen */ - canvas?: OffscreenCanvas | HTMLCanvasElement | null, + canvas?: OffscreenCanvas | HTMLCanvasElement | null | undefined, /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ readonly timestamp: number, /** getter property that returns unified persons object */ diff --git a/src/warmup.ts b/src/warmup.ts index aa69bc3b..8dc99f02 100644 --- a/src/warmup.ts +++ b/src/warmup.ts @@ -1,8 +1,10 @@ import { log, now, mergeDeep } from './helpers'; import * as sample from './sample'; import * as tf from '../dist/tfjs.esm.js'; -import { Config } from './config'; -import { Result } from './result'; +import * as image from './image/image'; +import type { Config } from './config'; +import type { Result } from './result'; +import { env } from './env'; async function warmupBitmap(instance) { const b64toBlob = (base64: string, type = 'application/octet-stream') => fetch(`data:${type};base64,${base64}`).then((res) => res.blob()); @@ -24,31 +26,38 @@ async function warmupBitmap(instance) { async function warmupCanvas(instance) { return new Promise((resolve) => { let src; - let size = 0; + // let size = 0; switch (instance.config.warmup) { case 'face': - size = 256; + // size = 256; src = 'data:image/jpeg;base64,' + sample.face; break; case 'full': case 'body': - size = 1200; + // size = 1200; src = 'data:image/jpeg;base64,' + sample.body; break; default: src = null; } // src = encodeURI('../assets/human-sample-upper.jpg'); - const img = new Image(); + let img; + if (typeof Image !== 'undefined') img = new Image(); + // @ts-ignore env.image is an external monkey-patch + else if (env.Image) img = new env.Image(); img.onload = async () => { - const canvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(size, size) : document.createElement('canvas'); - canvas.width = img.naturalWidth; - canvas.height = img.naturalHeight; - const ctx = canvas.getContext('2d'); - ctx?.drawImage(img, 0, 0); - // const data = ctx?.getImageData(0, 0, canvas.height, canvas.width); - const res = await instance.detect(canvas, instance.config); - resolve(res); + const canvas = image.canvas(img.naturalWidth, img.naturalHeight); + if (!canvas) { + log('Warmup: Canvas not found'); + resolve({}); + } else { + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0); + // const data = ctx?.getImageData(0, 0, canvas.height, canvas.width); + const tensor = await instance.image(canvas); + const res = await instance.detect(tensor.tensor, instance.config); + resolve(res); + } }; if (src) img.src = src; else resolve(null); @@ -93,7 +102,7 @@ export async function warmup(instance, userConfig?: Partial): Promise { - log.info(test, 'start'); const child = fork(path.join(__dirname, test), [], { silent: true }); child.on('message', (data) => logMessage(test, data)); child.on('error', (data) => log.error(test, ':', data.message || data)); @@ -68,6 +71,7 @@ async function testAll() { process.on('uncaughtException', (data) => log.error('nodejs unhandled exception', data)); log.info('tests:', tests); for (const test of tests) await runTest(test); + log.info(); log.info('status:', status); }