diff --git a/README.md b/README.md
index 12ee8cfd..4088690d 100644
--- a/README.md
+++ b/README.md
@@ -195,11 +195,11 @@ draw output on screen using internal draw helper functions
// create instance of human with simple configuration using default values
const config = { backend: 'webgl' };
const human = new Human(config);
+// select input HTMLVideoElement and output HTMLCanvasElement from page
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
function detectVideo() {
- // select input HTMLVideoElement and output HTMLCanvasElement from page
- const inputVideo = document.getElementById('video-id');
- const outputCanvas = document.getElementById('canvas-id');
// perform processing using default configuration
human.detect(inputVideo).then((result) => {
// result object will contain detected details
@@ -225,10 +225,10 @@ or using `async/await`:
// create instance of human with simple configuration using default values
const config = { backend: 'webgl' };
const human = new Human(config); // create instance of Human
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
async function detectVideo() {
- const inputVideo = document.getElementById('video-id');
- const outputCanvas = document.getElementById('canvas-id');
const result = await human.detect(inputVideo); // run detection
human.draw.all(outputCanvas, result); // draw all results
requestAnimationFrame(detectVideo); // run loop
@@ -237,6 +237,27 @@ async function detectVideo() {
detectVideo(); // start loop
```
+or using `Events`:
+
+```js
+// create instance of human with simple configuration using default values
+const config = { backend: 'webgl' };
+const human = new Human(config); // create instance of Human
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
+
+human.events.addEventListener('detect', () => { // event gets triggered when detect is complete
+ human.draw.all(outputCanvas, human.result); // draw all results
+});
+
+function detectVideo() {
+ human.detect(inputVideo) // run detection
+ .then(() => requestAnimationFrame(detectVideo)); // upon detect complete start processing of the next frame
+}
+
+detectVideo(); // start loop
+```
+
or using interpolated results for smooth video processing by separating detection and drawing loops:
```js
diff --git a/demo/nodejs/node-canvas.js b/demo/nodejs/node-canvas.js
index f95873f8..8b80c42f 100644
--- a/demo/nodejs/node-canvas.js
+++ b/demo/nodejs/node-canvas.js
@@ -7,8 +7,6 @@ const fs = require('fs');
const process = require('process');
const canvas = require('canvas');
-let fetch; // fetch is dynamically imported later
-
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
@@ -51,25 +49,26 @@ async function init() {
}
async function detect(input, output) {
- // read input image file and create tensor to be used for processing
+ // read input image from file or url into buffer
let buffer;
log.info('Loading image:', input);
if (input.startsWith('http:') || input.startsWith('https:')) {
+ const fetch = (await import('node-fetch')).default;
const res = await fetch(input);
if (res && res.ok) buffer = await res.buffer();
else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
} else {
buffer = fs.readFileSync(input);
}
+ if (!buffer) return {};
// decode image using tfjs-node so we don't need external depenencies
- // can also be done using canvas.js or some other 3rd party image library
- if (!buffer) return {};
+ /*
const tensor = human.tf.tidy(() => {
const decode = human.tf.node.decodeImage(buffer, 3);
let expand;
if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
- const channels = human.tf.split(decode, 4, 2); // tf.split(tensor, 4, 2); // split rgba to channels
+ const channels = human.tf.split(decode, 4, 2); // split rgba to channels
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
} else {
@@ -78,6 +77,22 @@ async function detect(input, output) {
const cast = human.tf.cast(expand, 'float32');
return cast;
});
+ */
+
+ // decode image using canvas library
+ const inputImage = await canvas.loadImage(input);
+ const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image');
+ const inputCtx = inputCanvas.getContext('2d');
+ inputCtx.drawImage(inputImage, 0, 0);
+ const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height);
+ const tensor = human.tf.tidy(() => {
+ const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]);
+ const channels = human.tf.split(data, 4, 2); // split rgba to channels
+ const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
+ const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
+ const cast = human.tf.cast(expand, 'float32');
+ return cast;
+ });
// image shape contains image dimensions and depth
log.state('Processing:', tensor['shape']);
@@ -130,7 +145,6 @@ async function detect(input, output) {
async function main() {
log.header();
log.info('Current folder:', process.env.PWD);
- fetch = (await import('node-fetch')).default;
await init();
const input = process.argv[2];
const output = process.argv[3];
diff --git a/demo/nodejs/node-event.js b/demo/nodejs/node-event.js
new file mode 100644
index 00000000..f8eb6a77
--- /dev/null
+++ b/demo/nodejs/node-event.js
@@ -0,0 +1,110 @@
+/**
+ * Human demo for NodeJS
+ */
+
+const log = require('@vladmandic/pilogger');
+const fs = require('fs');
+const process = require('process');
+
+let fetch; // fetch is dynamically imported later
+
+// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
+// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
+
+// load specific version of Human library that matches TensorFlow mode
+const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default;
+
+let human = null;
+
+const myConfig = {
+ backend: 'tensorflow',
+ modelBasePath: 'file://models/',
+ debug: false,
+ async: true,
+ filter: { enabled: false },
+ face: {
+ enabled: true,
+ detector: { enabled: true },
+ mesh: { enabled: true },
+ iris: { enabled: true },
+ description: { enabled: true },
+ emotion: { enabled: true },
+ },
+ hand: { enabled: true },
+ body: { enabled: true },
+ object: { enabled: true },
+};
+
+async function detect(input) {
+ // read input image from file or url into buffer
+ let buffer;
+ log.info('Loading image:', input);
+ if (input.startsWith('http:') || input.startsWith('https:')) {
+ fetch = (await import('node-fetch')).default;
+ const res = await fetch(input);
+ if (res && res.ok) buffer = await res.buffer();
+ else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
+ } else {
+ buffer = fs.readFileSync(input);
+ }
+
+ // decode image using tfjs-node so we don't need external depenencies
+ if (!buffer) return;
+ const tensor = human.tf.tidy(() => {
+ const decode = human.tf.node.decodeImage(buffer, 3);
+ let expand;
+ if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
+ const channels = human.tf.split(decode, 4, 2); // split rgba to channels
+ const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
+ expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
+ } else {
+ expand = human.tf.expandDims(decode, 0);
+ }
+ const cast = human.tf.cast(expand, 'float32');
+ return cast;
+ });
+
+ // run detection
+ await human.detect(tensor, myConfig);
+ human.tf.dispose(tensor); // dispose image tensor as we no longer need it
+}
+
+async function main() {
+ log.header();
+
+ human = new Human(myConfig);
+
+ human.events.addEventListener('warmup', () => {
+ log.info('Event Warmup');
+ });
+
+ human.events.addEventListener('load', () => {
+ const loaded = Object.keys(human.models).filter((a) => human.models[a]);
+ log.info('Event Loaded:', loaded, human.tf.engine().memory());
+ });
+
+ human.events.addEventListener('image', () => {
+ log.info('Event Image:', human.process.tensor.shape);
+ });
+
+ human.events.addEventListener('detect', () => {
+ log.data('Event Detected:');
+ const persons = human.result.persons;
+ for (let i = 0; i < persons.length; i++) {
+ const face = persons[i].face;
+ const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
+ const body = persons[i].body;
+ const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
+ log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
+ }
+ });
+
+ await human.tf.ready(); // wait until tf is ready
+
+ const input = process.argv[2]; // process input
+ if (input) await detect(input);
+ else log.error('Missing ');
+}
+
+main();
diff --git a/src/human.ts b/src/human.ts
index 0a04864e..e3b38eb8 100644
--- a/src/human.ts
+++ b/src/human.ts
@@ -76,8 +76,10 @@ export class Human {
* - Progresses through: 'config', 'check', 'backend', 'load', 'run:', 'idle'
*/
state: string;
- /** @internal: Instance of current image being processed */
- image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
+ /** process input and return tensor and canvas */
+ image: typeof image.process;
+ /** currenty processed image tensor and canvas */
+ process: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
/** @internal: Instance of TensorFlow/JS used by Human
* - Can be embedded or externally provided
*/
@@ -87,7 +89,7 @@ export class Human {
* - face: draw detected faces
* - body: draw detected people and body parts
* - hand: draw detected hands and hand parts
- * - canvas: draw processed canvas which is a processed copy of the input
+ * - canvas: draw this.processed canvas which is a this.processed copy of the input
* - all: meta-function that performs: canvas, face, body, hand
*/
draw: {
@@ -126,6 +128,17 @@ export class Human {
faceres: GraphModel | null,
segmentation: GraphModel | null,
};
+ /** Container for events dispatched by Human
+ *
+ * Possible events:
+ * - `create`: triggered when Human object is instantiated
+ * - `load`: triggered when models are loaded (explicitly or on-demand)
+ * - `image`: triggered when input image is this.processed
+ * - `result`: triggered when detection is complete
+ * - `warmup`: triggered when warmup is complete
+ */
+
+ events: EventTarget;
/** Reference face triangualtion array of 468 points, used for triangle references between points */
faceTriangulation: typeof facemesh.triangulation;
/** Refernce UV map of 468 values, used for 3D mapping of the face mesh */
@@ -161,6 +174,7 @@ export class Human {
this.#firstRun = true;
this.#lastCacheDiff = 0;
this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 };
+ this.events = new EventTarget();
// object that contains all initialized models
this.models = {
face: null,
@@ -179,15 +193,17 @@ export class Human {
segmentation: null,
};
this.result = { face: [], body: [], hand: [], gesture: [], object: [], performance: {}, timestamp: 0, persons: [] };
- // export access to image processing
+ // export access to image this.processing
// @ts-ignore eslint-typescript cannot correctly infer type in anonymous function
this.image = (input: Input) => image.process(input, this.config);
+ this.process = { tensor: null, canvas: null };
// export raw access to underlying models
this.faceTriangulation = facemesh.triangulation;
this.faceUVMap = facemesh.uvmap;
// include platform info
this.sysinfo = sysinfo.info();
this.#lastInputSum = 1;
+ this.#emit('create');
}
// helper function: measure tensor leak
@@ -228,9 +244,9 @@ export class Human {
}
/**
- * Segmentation method takes any input and returns processed canvas with body segmentation
+ * Segmentation method takes any input and returns this.processed canvas with body segmentation
* Optional parameter background is used to fill the background with specific input
- * Segmentation is not triggered as part of detect process
+ * Segmentation is not triggered as part of detect this.process
*
* @param input: {@link Input}
* @param background?: {@link Input}
@@ -240,7 +256,7 @@ export class Human {
return segmentation.process(input, background, this.config);
}
- /** Enhance method performs additional enhacements to face image previously detected for futher processing
+ /** Enhance method performs additional enhacements to face image previously detected for futher this.processing
* @param input: Tensor as provided in human.result.face[n].tensor
* @returns Tensor
*/
@@ -267,6 +283,7 @@ export class Human {
async load(userConfig?: Config | Record) {
this.state = 'load';
const timeStamp = now();
+ const count = Object.values(this.models).filter((model) => model).length;
if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config;
if (this.#firstRun) { // print version info on first run and check for correct backend setup
@@ -289,10 +306,16 @@ export class Human {
this.#firstRun = false;
}
+ const loaded = Object.values(this.models).filter((model) => model).length;
+ if (loaded !== count) this.#emit('load');
const current = Math.trunc(now() - timeStamp);
if (current > (this.performance.load as number || 0)) this.performance.load = current;
}
+ // emit event
+ /** @hidden */
+ #emit = (event: string) => this.events?.dispatchEvent(new Event(event));
+
// check if backend needs initialization if it changed
/** @hidden */
#checkBackend = async (force = false) => {
@@ -433,9 +456,9 @@ export class Human {
/** Main detection method
* - Analyze configuration: {@link Config}
- * - Pre-process input: {@link Input}
+ * - Pre-this.process input: {@link Input}
* - Run inference for all configured models
- * - Process and return result: {@link Result}
+ * - this.process and return result: {@link Result}
*
* @param input: Input
* @param userConfig?: {@link Config}
@@ -468,34 +491,35 @@ export class Human {
await this.load();
timeStamp = now();
- let process = image.process(input, this.config);
+ this.process = image.process(input, this.config);
this.performance.image = Math.trunc(now() - timeStamp);
this.analyze('Get Image:');
- // run segmentation preprocessing
- if (this.config.segmentation.enabled && process && process.tensor) {
+ // run segmentation prethis.processing
+ if (this.config.segmentation.enabled && this.process && this.process.tensor) {
this.analyze('Start Segmentation:');
this.state = 'run:segmentation';
timeStamp = now();
- await segmentation.predict(process);
+ await segmentation.predict(this.process);
elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
- if (process.canvas) {
+ if (this.process.canvas) {
// replace input
- tf.dispose(process.tensor);
- process = image.process(process.canvas, this.config);
+ tf.dispose(this.process.tensor);
+ this.process = image.process(this.process.canvas, this.config);
}
this.analyze('End Segmentation:');
}
- if (!process || !process.tensor) {
+ if (!this.process || !this.process.tensor) {
log('could not convert input to tensor');
resolve({ error: 'could not convert input to tensor' });
return;
}
+ this.#emit('image');
timeStamp = now();
- this.config.skipFrame = await this.#skipFrame(process.tensor);
+ this.config.skipFrame = await this.#skipFrame(this.process.tensor);
if (!this.performance.frames) this.performance.frames = 0;
if (!this.performance.cached) this.performance.cached = 0;
(this.performance.frames as number)++;
@@ -512,12 +536,12 @@ export class Human {
// run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
if (this.config.async) {
- faceRes = this.config.face.enabled ? face.detectFace(this, process.tensor) : [];
+ faceRes = this.config.face.enabled ? face.detectFace(this, this.process.tensor) : [];
if (this.performance.face) delete this.performance.face;
} else {
this.state = 'run:face';
timeStamp = now();
- faceRes = this.config.face.enabled ? await face.detectFace(this, process.tensor) : [];
+ faceRes = this.config.face.enabled ? await face.detectFace(this, this.process.tensor) : [];
elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.performance.face = elapsedTime;
}
@@ -525,18 +549,18 @@ export class Human {
// run body: can be posenet, blazepose, efficientpose, movenet
this.analyze('Start Body:');
if (this.config.async) {
- if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(process.tensor, this.config) : [];
+ if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : [];
if (this.performance.body) delete this.performance.body;
} else {
this.state = 'run:body';
timeStamp = now();
- if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
- else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(process.tensor, this.config) : [];
+ if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : [];
+ else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : [];
elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.performance.body = elapsedTime;
}
@@ -545,12 +569,12 @@ export class Human {
// run handpose
this.analyze('Start Hand:');
if (this.config.async) {
- handRes = this.config.hand.enabled ? handpose.predict(process.tensor, this.config) : [];
+ handRes = this.config.hand.enabled ? handpose.predict(this.process.tensor, this.config) : [];
if (this.performance.hand) delete this.performance.hand;
} else {
this.state = 'run:hand';
timeStamp = now();
- handRes = this.config.hand.enabled ? await handpose.predict(process.tensor, this.config) : [];
+ handRes = this.config.hand.enabled ? await handpose.predict(this.process.tensor, this.config) : [];
elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.performance.hand = elapsedTime;
}
@@ -559,14 +583,14 @@ export class Human {
// run nanodet
this.analyze('Start Object:');
if (this.config.async) {
- if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
- else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : [];
+ if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : [];
+ else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : [];
if (this.performance.object) delete this.performance.object;
} else {
this.state = 'run:object';
timeStamp = now();
- if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
- else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : [];
+ if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : [];
+ else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : [];
elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.performance.object = elapsedTime;
}
@@ -586,6 +610,7 @@ export class Human {
this.performance.total = Math.trunc(now() - timeStart);
this.state = 'idle';
+ const shape = this.process?.tensor?.shape || [];
this.result = {
face: faceRes as Face[],
body: bodyRes as Body[],
@@ -593,15 +618,16 @@ export class Human {
gesture: gestureRes,
object: objectRes as Item[],
performance: this.performance,
- canvas: process.canvas,
+ canvas: this.process.canvas,
timestamp: Date.now(),
- get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, process?.tensor?.shape); },
+ get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); },
};
// finally dispose input tensor
- tf.dispose(process.tensor);
+ tf.dispose(this.process.tensor);
// log('Result:', result);
+ this.#emit('detect');
resolve(this.result);
});
}
@@ -700,6 +726,7 @@ export class Human {
else res = await this.#warmupNode();
const t1 = now();
if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res);
+ this.#emit('warmup');
return res;
}
}
diff --git a/src/result.ts b/src/result.ts
index 44eaadbe..ec7cf290 100644
--- a/src/result.ts
+++ b/src/result.ts
@@ -186,7 +186,7 @@ export interface Result {
/** global performance object with timing values for each operation */
performance: Record,
/** optional processed canvas that can be used to draw input on screen */
- canvas?: OffscreenCanvas | HTMLCanvasElement,
+ canvas?: OffscreenCanvas | HTMLCanvasElement | null,
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
readonly timestamp: number,
/** getter property that returns unified persons object */
diff --git a/src/warmup.ts b/src/warmup.ts
deleted file mode 100644
index e69de29b..00000000