From 34a3a42fba81bf95bc2aff08ae974663e329dd6f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 11 Sep 2021 16:00:16 -0400 Subject: [PATCH] implement event emitters --- README.md | 31 +++++++++-- demo/nodejs/node-canvas.js | 28 +++++++--- demo/nodejs/node-event.js | 110 +++++++++++++++++++++++++++++++++++++ src/human.ts | 101 +++++++++++++++++++++------------- src/result.ts | 2 +- src/warmup.ts | 0 6 files changed, 222 insertions(+), 50 deletions(-) create mode 100644 demo/nodejs/node-event.js delete mode 100644 src/warmup.ts diff --git a/README.md b/README.md index 12ee8cfd..4088690d 100644 --- a/README.md +++ b/README.md @@ -195,11 +195,11 @@ draw output on screen using internal draw helper functions // create instance of human with simple configuration using default values const config = { backend: 'webgl' }; const human = new Human(config); +// select input HTMLVideoElement and output HTMLCanvasElement from page +const inputVideo = document.getElementById('video-id'); +const outputCanvas = document.getElementById('canvas-id'); function detectVideo() { - // select input HTMLVideoElement and output HTMLCanvasElement from page - const inputVideo = document.getElementById('video-id'); - const outputCanvas = document.getElementById('canvas-id'); // perform processing using default configuration human.detect(inputVideo).then((result) => { // result object will contain detected details @@ -225,10 +225,10 @@ or using `async/await`: // create instance of human with simple configuration using default values const config = { backend: 'webgl' }; const human = new Human(config); // create instance of Human +const inputVideo = document.getElementById('video-id'); +const outputCanvas = document.getElementById('canvas-id'); async function detectVideo() { - const inputVideo = document.getElementById('video-id'); - const outputCanvas = document.getElementById('canvas-id'); const result = await human.detect(inputVideo); // run detection human.draw.all(outputCanvas, result); // draw all results requestAnimationFrame(detectVideo); // run loop @@ -237,6 +237,27 @@ async function detectVideo() { detectVideo(); // start loop ``` +or using `Events`: + +```js +// create instance of human with simple configuration using default values +const config = { backend: 'webgl' }; +const human = new Human(config); // create instance of Human +const inputVideo = document.getElementById('video-id'); +const outputCanvas = document.getElementById('canvas-id'); + +human.events.addEventListener('detect', () => { // event gets triggered when detect is complete + human.draw.all(outputCanvas, human.result); // draw all results +}); + +function detectVideo() { + human.detect(inputVideo) // run detection + .then(() => requestAnimationFrame(detectVideo)); // upon detect complete start processing of the next frame +} + +detectVideo(); // start loop +``` + or using interpolated results for smooth video processing by separating detection and drawing loops: ```js diff --git a/demo/nodejs/node-canvas.js b/demo/nodejs/node-canvas.js index f95873f8..8b80c42f 100644 --- a/demo/nodejs/node-canvas.js +++ b/demo/nodejs/node-canvas.js @@ -7,8 +7,6 @@ const fs = require('fs'); const process = require('process'); const canvas = require('canvas'); -let fetch; // fetch is dynamically imported later - // for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu'); @@ -51,25 +49,26 @@ async function init() { } async function detect(input, output) { - // read input image file and create tensor to be used for processing + // read input image from file or url into buffer let buffer; log.info('Loading image:', input); if (input.startsWith('http:') || input.startsWith('https:')) { + const fetch = (await import('node-fetch')).default; const res = await fetch(input); if (res && res.ok) buffer = await res.buffer(); else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type')); } else { buffer = fs.readFileSync(input); } + if (!buffer) return {}; // decode image using tfjs-node so we don't need external depenencies - // can also be done using canvas.js or some other 3rd party image library - if (!buffer) return {}; + /* const tensor = human.tf.tidy(() => { const decode = human.tf.node.decodeImage(buffer, 3); let expand; if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb - const channels = human.tf.split(decode, 4, 2); // tf.split(tensor, 4, 2); // split rgba to channels + const channels = human.tf.split(decode, 4, 2); // split rgba to channels const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead } else { @@ -78,6 +77,22 @@ async function detect(input, output) { const cast = human.tf.cast(expand, 'float32'); return cast; }); + */ + + // decode image using canvas library + const inputImage = await canvas.loadImage(input); + const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image'); + const inputCtx = inputCanvas.getContext('2d'); + inputCtx.drawImage(inputImage, 0, 0); + const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height); + const tensor = human.tf.tidy(() => { + const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]); + const channels = human.tf.split(data, 4, 2); // split rgba to channels + const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha + const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead + const cast = human.tf.cast(expand, 'float32'); + return cast; + }); // image shape contains image dimensions and depth log.state('Processing:', tensor['shape']); @@ -130,7 +145,6 @@ async function detect(input, output) { async function main() { log.header(); log.info('Current folder:', process.env.PWD); - fetch = (await import('node-fetch')).default; await init(); const input = process.argv[2]; const output = process.argv[3]; diff --git a/demo/nodejs/node-event.js b/demo/nodejs/node-event.js new file mode 100644 index 00000000..f8eb6a77 --- /dev/null +++ b/demo/nodejs/node-event.js @@ -0,0 +1,110 @@ +/** + * Human demo for NodeJS + */ + +const log = require('@vladmandic/pilogger'); +const fs = require('fs'); +const process = require('process'); + +let fetch; // fetch is dynamically imported later + +// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu'); + +// load specific version of Human library that matches TensorFlow mode +const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default; + +let human = null; + +const myConfig = { + backend: 'tensorflow', + modelBasePath: 'file://models/', + debug: false, + async: true, + filter: { enabled: false }, + face: { + enabled: true, + detector: { enabled: true }, + mesh: { enabled: true }, + iris: { enabled: true }, + description: { enabled: true }, + emotion: { enabled: true }, + }, + hand: { enabled: true }, + body: { enabled: true }, + object: { enabled: true }, +}; + +async function detect(input) { + // read input image from file or url into buffer + let buffer; + log.info('Loading image:', input); + if (input.startsWith('http:') || input.startsWith('https:')) { + fetch = (await import('node-fetch')).default; + const res = await fetch(input); + if (res && res.ok) buffer = await res.buffer(); + else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type')); + } else { + buffer = fs.readFileSync(input); + } + + // decode image using tfjs-node so we don't need external depenencies + if (!buffer) return; + const tensor = human.tf.tidy(() => { + const decode = human.tf.node.decodeImage(buffer, 3); + let expand; + if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb + const channels = human.tf.split(decode, 4, 2); // split rgba to channels + const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha + expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead + } else { + expand = human.tf.expandDims(decode, 0); + } + const cast = human.tf.cast(expand, 'float32'); + return cast; + }); + + // run detection + await human.detect(tensor, myConfig); + human.tf.dispose(tensor); // dispose image tensor as we no longer need it +} + +async function main() { + log.header(); + + human = new Human(myConfig); + + human.events.addEventListener('warmup', () => { + log.info('Event Warmup'); + }); + + human.events.addEventListener('load', () => { + const loaded = Object.keys(human.models).filter((a) => human.models[a]); + log.info('Event Loaded:', loaded, human.tf.engine().memory()); + }); + + human.events.addEventListener('image', () => { + log.info('Event Image:', human.process.tensor.shape); + }); + + human.events.addEventListener('detect', () => { + log.data('Event Detected:'); + const persons = human.result.persons; + for (let i = 0; i < persons.length; i++) { + const face = persons[i].face; + const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null; + const body = persons[i].body; + const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null; + log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`); + } + }); + + await human.tf.ready(); // wait until tf is ready + + const input = process.argv[2]; // process input + if (input) await detect(input); + else log.error('Missing '); +} + +main(); diff --git a/src/human.ts b/src/human.ts index 0a04864e..e3b38eb8 100644 --- a/src/human.ts +++ b/src/human.ts @@ -76,8 +76,10 @@ export class Human { * - Progresses through: 'config', 'check', 'backend', 'load', 'run:', 'idle' */ state: string; - /** @internal: Instance of current image being processed */ - image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null }; + /** process input and return tensor and canvas */ + image: typeof image.process; + /** currenty processed image tensor and canvas */ + process: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null }; /** @internal: Instance of TensorFlow/JS used by Human * - Can be embedded or externally provided */ @@ -87,7 +89,7 @@ export class Human { * - face: draw detected faces * - body: draw detected people and body parts * - hand: draw detected hands and hand parts - * - canvas: draw processed canvas which is a processed copy of the input + * - canvas: draw this.processed canvas which is a this.processed copy of the input * - all: meta-function that performs: canvas, face, body, hand */ draw: { @@ -126,6 +128,17 @@ export class Human { faceres: GraphModel | null, segmentation: GraphModel | null, }; + /** Container for events dispatched by Human + * + * Possible events: + * - `create`: triggered when Human object is instantiated + * - `load`: triggered when models are loaded (explicitly or on-demand) + * - `image`: triggered when input image is this.processed + * - `result`: triggered when detection is complete + * - `warmup`: triggered when warmup is complete + */ + + events: EventTarget; /** Reference face triangualtion array of 468 points, used for triangle references between points */ faceTriangulation: typeof facemesh.triangulation; /** Refernce UV map of 468 values, used for 3D mapping of the face mesh */ @@ -161,6 +174,7 @@ export class Human { this.#firstRun = true; this.#lastCacheDiff = 0; this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 }; + this.events = new EventTarget(); // object that contains all initialized models this.models = { face: null, @@ -179,15 +193,17 @@ export class Human { segmentation: null, }; this.result = { face: [], body: [], hand: [], gesture: [], object: [], performance: {}, timestamp: 0, persons: [] }; - // export access to image processing + // export access to image this.processing // @ts-ignore eslint-typescript cannot correctly infer type in anonymous function this.image = (input: Input) => image.process(input, this.config); + this.process = { tensor: null, canvas: null }; // export raw access to underlying models this.faceTriangulation = facemesh.triangulation; this.faceUVMap = facemesh.uvmap; // include platform info this.sysinfo = sysinfo.info(); this.#lastInputSum = 1; + this.#emit('create'); } // helper function: measure tensor leak @@ -228,9 +244,9 @@ export class Human { } /** - * Segmentation method takes any input and returns processed canvas with body segmentation + * Segmentation method takes any input and returns this.processed canvas with body segmentation * Optional parameter background is used to fill the background with specific input - * Segmentation is not triggered as part of detect process + * Segmentation is not triggered as part of detect this.process * * @param input: {@link Input} * @param background?: {@link Input} @@ -240,7 +256,7 @@ export class Human { return segmentation.process(input, background, this.config); } - /** Enhance method performs additional enhacements to face image previously detected for futher processing + /** Enhance method performs additional enhacements to face image previously detected for futher this.processing * @param input: Tensor as provided in human.result.face[n].tensor * @returns Tensor */ @@ -267,6 +283,7 @@ export class Human { async load(userConfig?: Config | Record) { this.state = 'load'; const timeStamp = now(); + const count = Object.values(this.models).filter((model) => model).length; if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config; if (this.#firstRun) { // print version info on first run and check for correct backend setup @@ -289,10 +306,16 @@ export class Human { this.#firstRun = false; } + const loaded = Object.values(this.models).filter((model) => model).length; + if (loaded !== count) this.#emit('load'); const current = Math.trunc(now() - timeStamp); if (current > (this.performance.load as number || 0)) this.performance.load = current; } + // emit event + /** @hidden */ + #emit = (event: string) => this.events?.dispatchEvent(new Event(event)); + // check if backend needs initialization if it changed /** @hidden */ #checkBackend = async (force = false) => { @@ -433,9 +456,9 @@ export class Human { /** Main detection method * - Analyze configuration: {@link Config} - * - Pre-process input: {@link Input} + * - Pre-this.process input: {@link Input} * - Run inference for all configured models - * - Process and return result: {@link Result} + * - this.process and return result: {@link Result} * * @param input: Input * @param userConfig?: {@link Config} @@ -468,34 +491,35 @@ export class Human { await this.load(); timeStamp = now(); - let process = image.process(input, this.config); + this.process = image.process(input, this.config); this.performance.image = Math.trunc(now() - timeStamp); this.analyze('Get Image:'); - // run segmentation preprocessing - if (this.config.segmentation.enabled && process && process.tensor) { + // run segmentation prethis.processing + if (this.config.segmentation.enabled && this.process && this.process.tensor) { this.analyze('Start Segmentation:'); this.state = 'run:segmentation'; timeStamp = now(); - await segmentation.predict(process); + await segmentation.predict(this.process); elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.segmentation = elapsedTime; - if (process.canvas) { + if (this.process.canvas) { // replace input - tf.dispose(process.tensor); - process = image.process(process.canvas, this.config); + tf.dispose(this.process.tensor); + this.process = image.process(this.process.canvas, this.config); } this.analyze('End Segmentation:'); } - if (!process || !process.tensor) { + if (!this.process || !this.process.tensor) { log('could not convert input to tensor'); resolve({ error: 'could not convert input to tensor' }); return; } + this.#emit('image'); timeStamp = now(); - this.config.skipFrame = await this.#skipFrame(process.tensor); + this.config.skipFrame = await this.#skipFrame(this.process.tensor); if (!this.performance.frames) this.performance.frames = 0; if (!this.performance.cached) this.performance.cached = 0; (this.performance.frames as number)++; @@ -512,12 +536,12 @@ export class Human { // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion if (this.config.async) { - faceRes = this.config.face.enabled ? face.detectFace(this, process.tensor) : []; + faceRes = this.config.face.enabled ? face.detectFace(this, this.process.tensor) : []; if (this.performance.face) delete this.performance.face; } else { this.state = 'run:face'; timeStamp = now(); - faceRes = this.config.face.enabled ? await face.detectFace(this, process.tensor) : []; + faceRes = this.config.face.enabled ? await face.detectFace(this, this.process.tensor) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.face = elapsedTime; } @@ -525,18 +549,18 @@ export class Human { // run body: can be posenet, blazepose, efficientpose, movenet this.analyze('Start Body:'); if (this.config.async) { - if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(process.tensor, this.config) : []; + if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : []; if (this.performance.body) delete this.performance.body; } else { this.state = 'run:body'; timeStamp = now(); - if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(process.tensor, this.config) : []; + if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.body = elapsedTime; } @@ -545,12 +569,12 @@ export class Human { // run handpose this.analyze('Start Hand:'); if (this.config.async) { - handRes = this.config.hand.enabled ? handpose.predict(process.tensor, this.config) : []; + handRes = this.config.hand.enabled ? handpose.predict(this.process.tensor, this.config) : []; if (this.performance.hand) delete this.performance.hand; } else { this.state = 'run:hand'; timeStamp = now(); - handRes = this.config.hand.enabled ? await handpose.predict(process.tensor, this.config) : []; + handRes = this.config.hand.enabled ? await handpose.predict(this.process.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.hand = elapsedTime; } @@ -559,14 +583,14 @@ export class Human { // run nanodet this.analyze('Start Object:'); if (this.config.async) { - if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : []; - else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : []; + if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : []; + else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : []; if (this.performance.object) delete this.performance.object; } else { this.state = 'run:object'; timeStamp = now(); - if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : []; - else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : []; + if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : []; + else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.object = elapsedTime; } @@ -586,6 +610,7 @@ export class Human { this.performance.total = Math.trunc(now() - timeStart); this.state = 'idle'; + const shape = this.process?.tensor?.shape || []; this.result = { face: faceRes as Face[], body: bodyRes as Body[], @@ -593,15 +618,16 @@ export class Human { gesture: gestureRes, object: objectRes as Item[], performance: this.performance, - canvas: process.canvas, + canvas: this.process.canvas, timestamp: Date.now(), - get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, process?.tensor?.shape); }, + get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); }, }; // finally dispose input tensor - tf.dispose(process.tensor); + tf.dispose(this.process.tensor); // log('Result:', result); + this.#emit('detect'); resolve(this.result); }); } @@ -700,6 +726,7 @@ export class Human { else res = await this.#warmupNode(); const t1 = now(); if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res); + this.#emit('warmup'); return res; } } diff --git a/src/result.ts b/src/result.ts index 44eaadbe..ec7cf290 100644 --- a/src/result.ts +++ b/src/result.ts @@ -186,7 +186,7 @@ export interface Result { /** global performance object with timing values for each operation */ performance: Record, /** optional processed canvas that can be used to draw input on screen */ - canvas?: OffscreenCanvas | HTMLCanvasElement, + canvas?: OffscreenCanvas | HTMLCanvasElement | null, /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ readonly timestamp: number, /** getter property that returns unified persons object */ diff --git a/src/warmup.ts b/src/warmup.ts deleted file mode 100644 index e69de29b..00000000