mirror of https://github.com/vladmandic/human
implement event emitters
Vladimir Mandic
parent
cd77ccdef6
commit
34a3a42fba
31
README.md
31
README.md
|
|
@ -195,11 +195,11 @@ draw output on screen using internal draw helper functions
|
|||
// create instance of human with simple configuration using default values
|
||||
const config = { backend: 'webgl' };
|
||||
const human = new Human(config);
|
||||
// select input HTMLVideoElement and output HTMLCanvasElement from page
|
||||
const inputVideo = document.getElementById('video-id');
|
||||
const outputCanvas = document.getElementById('canvas-id');
|
||||
|
||||
function detectVideo() {
|
||||
// select input HTMLVideoElement and output HTMLCanvasElement from page
|
||||
const inputVideo = document.getElementById('video-id');
|
||||
const outputCanvas = document.getElementById('canvas-id');
|
||||
// perform processing using default configuration
|
||||
human.detect(inputVideo).then((result) => {
|
||||
// result object will contain detected details
|
||||
|
|
@ -225,10 +225,10 @@ or using `async/await`:
|
|||
// create instance of human with simple configuration using default values
|
||||
const config = { backend: 'webgl' };
|
||||
const human = new Human(config); // create instance of Human
|
||||
const inputVideo = document.getElementById('video-id');
|
||||
const outputCanvas = document.getElementById('canvas-id');
|
||||
|
||||
async function detectVideo() {
|
||||
const inputVideo = document.getElementById('video-id');
|
||||
const outputCanvas = document.getElementById('canvas-id');
|
||||
const result = await human.detect(inputVideo); // run detection
|
||||
human.draw.all(outputCanvas, result); // draw all results
|
||||
requestAnimationFrame(detectVideo); // run loop
|
||||
|
|
@ -237,6 +237,27 @@ async function detectVideo() {
|
|||
detectVideo(); // start loop
|
||||
```
|
||||
|
||||
or using `Events`:
|
||||
|
||||
```js
|
||||
// create instance of human with simple configuration using default values
|
||||
const config = { backend: 'webgl' };
|
||||
const human = new Human(config); // create instance of Human
|
||||
const inputVideo = document.getElementById('video-id');
|
||||
const outputCanvas = document.getElementById('canvas-id');
|
||||
|
||||
human.events.addEventListener('detect', () => { // event gets triggered when detect is complete
|
||||
human.draw.all(outputCanvas, human.result); // draw all results
|
||||
});
|
||||
|
||||
function detectVideo() {
|
||||
human.detect(inputVideo) // run detection
|
||||
.then(() => requestAnimationFrame(detectVideo)); // upon detect complete start processing of the next frame
|
||||
}
|
||||
|
||||
detectVideo(); // start loop
|
||||
```
|
||||
|
||||
or using interpolated results for smooth video processing by separating detection and drawing loops:
|
||||
|
||||
```js
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ const fs = require('fs');
|
|||
const process = require('process');
|
||||
const canvas = require('canvas');
|
||||
|
||||
let fetch; // fetch is dynamically imported later
|
||||
|
||||
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
|
||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
|
||||
|
|
@ -51,25 +49,26 @@ async function init() {
|
|||
}
|
||||
|
||||
async function detect(input, output) {
|
||||
// read input image file and create tensor to be used for processing
|
||||
// read input image from file or url into buffer
|
||||
let buffer;
|
||||
log.info('Loading image:', input);
|
||||
if (input.startsWith('http:') || input.startsWith('https:')) {
|
||||
const fetch = (await import('node-fetch')).default;
|
||||
const res = await fetch(input);
|
||||
if (res && res.ok) buffer = await res.buffer();
|
||||
else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
|
||||
} else {
|
||||
buffer = fs.readFileSync(input);
|
||||
}
|
||||
if (!buffer) return {};
|
||||
|
||||
// decode image using tfjs-node so we don't need external depenencies
|
||||
// can also be done using canvas.js or some other 3rd party image library
|
||||
if (!buffer) return {};
|
||||
/*
|
||||
const tensor = human.tf.tidy(() => {
|
||||
const decode = human.tf.node.decodeImage(buffer, 3);
|
||||
let expand;
|
||||
if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
|
||||
const channels = human.tf.split(decode, 4, 2); // tf.split(tensor, 4, 2); // split rgba to channels
|
||||
const channels = human.tf.split(decode, 4, 2); // split rgba to channels
|
||||
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
|
||||
expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
|
||||
} else {
|
||||
|
|
@ -78,6 +77,22 @@ async function detect(input, output) {
|
|||
const cast = human.tf.cast(expand, 'float32');
|
||||
return cast;
|
||||
});
|
||||
*/
|
||||
|
||||
// decode image using canvas library
|
||||
const inputImage = await canvas.loadImage(input);
|
||||
const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image');
|
||||
const inputCtx = inputCanvas.getContext('2d');
|
||||
inputCtx.drawImage(inputImage, 0, 0);
|
||||
const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height);
|
||||
const tensor = human.tf.tidy(() => {
|
||||
const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]);
|
||||
const channels = human.tf.split(data, 4, 2); // split rgba to channels
|
||||
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
|
||||
const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
|
||||
const cast = human.tf.cast(expand, 'float32');
|
||||
return cast;
|
||||
});
|
||||
|
||||
// image shape contains image dimensions and depth
|
||||
log.state('Processing:', tensor['shape']);
|
||||
|
|
@ -130,7 +145,6 @@ async function detect(input, output) {
|
|||
async function main() {
|
||||
log.header();
|
||||
log.info('Current folder:', process.env.PWD);
|
||||
fetch = (await import('node-fetch')).default;
|
||||
await init();
|
||||
const input = process.argv[2];
|
||||
const output = process.argv[3];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
/**
|
||||
* Human demo for NodeJS
|
||||
*/
|
||||
|
||||
const log = require('@vladmandic/pilogger');
|
||||
const fs = require('fs');
|
||||
const process = require('process');
|
||||
|
||||
let fetch; // fetch is dynamically imported later
|
||||
|
||||
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
|
||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
|
||||
|
||||
// load specific version of Human library that matches TensorFlow mode
|
||||
const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default;
|
||||
|
||||
let human = null;
|
||||
|
||||
const myConfig = {
|
||||
backend: 'tensorflow',
|
||||
modelBasePath: 'file://models/',
|
||||
debug: false,
|
||||
async: true,
|
||||
filter: { enabled: false },
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { enabled: true },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
description: { enabled: true },
|
||||
emotion: { enabled: true },
|
||||
},
|
||||
hand: { enabled: true },
|
||||
body: { enabled: true },
|
||||
object: { enabled: true },
|
||||
};
|
||||
|
||||
async function detect(input) {
|
||||
// read input image from file or url into buffer
|
||||
let buffer;
|
||||
log.info('Loading image:', input);
|
||||
if (input.startsWith('http:') || input.startsWith('https:')) {
|
||||
fetch = (await import('node-fetch')).default;
|
||||
const res = await fetch(input);
|
||||
if (res && res.ok) buffer = await res.buffer();
|
||||
else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
|
||||
} else {
|
||||
buffer = fs.readFileSync(input);
|
||||
}
|
||||
|
||||
// decode image using tfjs-node so we don't need external depenencies
|
||||
if (!buffer) return;
|
||||
const tensor = human.tf.tidy(() => {
|
||||
const decode = human.tf.node.decodeImage(buffer, 3);
|
||||
let expand;
|
||||
if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
|
||||
const channels = human.tf.split(decode, 4, 2); // split rgba to channels
|
||||
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
|
||||
expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
|
||||
} else {
|
||||
expand = human.tf.expandDims(decode, 0);
|
||||
}
|
||||
const cast = human.tf.cast(expand, 'float32');
|
||||
return cast;
|
||||
});
|
||||
|
||||
// run detection
|
||||
await human.detect(tensor, myConfig);
|
||||
human.tf.dispose(tensor); // dispose image tensor as we no longer need it
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log.header();
|
||||
|
||||
human = new Human(myConfig);
|
||||
|
||||
human.events.addEventListener('warmup', () => {
|
||||
log.info('Event Warmup');
|
||||
});
|
||||
|
||||
human.events.addEventListener('load', () => {
|
||||
const loaded = Object.keys(human.models).filter((a) => human.models[a]);
|
||||
log.info('Event Loaded:', loaded, human.tf.engine().memory());
|
||||
});
|
||||
|
||||
human.events.addEventListener('image', () => {
|
||||
log.info('Event Image:', human.process.tensor.shape);
|
||||
});
|
||||
|
||||
human.events.addEventListener('detect', () => {
|
||||
log.data('Event Detected:');
|
||||
const persons = human.result.persons;
|
||||
for (let i = 0; i < persons.length; i++) {
|
||||
const face = persons[i].face;
|
||||
const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
|
||||
const body = persons[i].body;
|
||||
const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
|
||||
log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
|
||||
}
|
||||
});
|
||||
|
||||
await human.tf.ready(); // wait until tf is ready
|
||||
|
||||
const input = process.argv[2]; // process input
|
||||
if (input) await detect(input);
|
||||
else log.error('Missing <input>');
|
||||
}
|
||||
|
||||
main();
|
||||
101
src/human.ts
101
src/human.ts
|
|
@ -76,8 +76,10 @@ export class Human {
|
|||
* - Progresses through: 'config', 'check', 'backend', 'load', 'run:<model>', 'idle'
|
||||
*/
|
||||
state: string;
|
||||
/** @internal: Instance of current image being processed */
|
||||
image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
|
||||
/** process input and return tensor and canvas */
|
||||
image: typeof image.process;
|
||||
/** currenty processed image tensor and canvas */
|
||||
process: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
|
||||
/** @internal: Instance of TensorFlow/JS used by Human
|
||||
* - Can be embedded or externally provided
|
||||
*/
|
||||
|
|
@ -87,7 +89,7 @@ export class Human {
|
|||
* - face: draw detected faces
|
||||
* - body: draw detected people and body parts
|
||||
* - hand: draw detected hands and hand parts
|
||||
* - canvas: draw processed canvas which is a processed copy of the input
|
||||
* - canvas: draw this.processed canvas which is a this.processed copy of the input
|
||||
* - all: meta-function that performs: canvas, face, body, hand
|
||||
*/
|
||||
draw: {
|
||||
|
|
@ -126,6 +128,17 @@ export class Human {
|
|||
faceres: GraphModel | null,
|
||||
segmentation: GraphModel | null,
|
||||
};
|
||||
/** Container for events dispatched by Human
|
||||
*
|
||||
* Possible events:
|
||||
* - `create`: triggered when Human object is instantiated
|
||||
* - `load`: triggered when models are loaded (explicitly or on-demand)
|
||||
* - `image`: triggered when input image is this.processed
|
||||
* - `result`: triggered when detection is complete
|
||||
* - `warmup`: triggered when warmup is complete
|
||||
*/
|
||||
|
||||
events: EventTarget;
|
||||
/** Reference face triangualtion array of 468 points, used for triangle references between points */
|
||||
faceTriangulation: typeof facemesh.triangulation;
|
||||
/** Refernce UV map of 468 values, used for 3D mapping of the face mesh */
|
||||
|
|
@ -161,6 +174,7 @@ export class Human {
|
|||
this.#firstRun = true;
|
||||
this.#lastCacheDiff = 0;
|
||||
this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 };
|
||||
this.events = new EventTarget();
|
||||
// object that contains all initialized models
|
||||
this.models = {
|
||||
face: null,
|
||||
|
|
@ -179,15 +193,17 @@ export class Human {
|
|||
segmentation: null,
|
||||
};
|
||||
this.result = { face: [], body: [], hand: [], gesture: [], object: [], performance: {}, timestamp: 0, persons: [] };
|
||||
// export access to image processing
|
||||
// export access to image this.processing
|
||||
// @ts-ignore eslint-typescript cannot correctly infer type in anonymous function
|
||||
this.image = (input: Input) => image.process(input, this.config);
|
||||
this.process = { tensor: null, canvas: null };
|
||||
// export raw access to underlying models
|
||||
this.faceTriangulation = facemesh.triangulation;
|
||||
this.faceUVMap = facemesh.uvmap;
|
||||
// include platform info
|
||||
this.sysinfo = sysinfo.info();
|
||||
this.#lastInputSum = 1;
|
||||
this.#emit('create');
|
||||
}
|
||||
|
||||
// helper function: measure tensor leak
|
||||
|
|
@ -228,9 +244,9 @@ export class Human {
|
|||
}
|
||||
|
||||
/**
|
||||
* Segmentation method takes any input and returns processed canvas with body segmentation
|
||||
* Segmentation method takes any input and returns this.processed canvas with body segmentation
|
||||
* Optional parameter background is used to fill the background with specific input
|
||||
* Segmentation is not triggered as part of detect process
|
||||
* Segmentation is not triggered as part of detect this.process
|
||||
*
|
||||
* @param input: {@link Input}
|
||||
* @param background?: {@link Input}
|
||||
|
|
@ -240,7 +256,7 @@ export class Human {
|
|||
return segmentation.process(input, background, this.config);
|
||||
}
|
||||
|
||||
/** Enhance method performs additional enhacements to face image previously detected for futher processing
|
||||
/** Enhance method performs additional enhacements to face image previously detected for futher this.processing
|
||||
* @param input: Tensor as provided in human.result.face[n].tensor
|
||||
* @returns Tensor
|
||||
*/
|
||||
|
|
@ -267,6 +283,7 @@ export class Human {
|
|||
async load(userConfig?: Config | Record<string, unknown>) {
|
||||
this.state = 'load';
|
||||
const timeStamp = now();
|
||||
const count = Object.values(this.models).filter((model) => model).length;
|
||||
if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config;
|
||||
|
||||
if (this.#firstRun) { // print version info on first run and check for correct backend setup
|
||||
|
|
@ -289,10 +306,16 @@ export class Human {
|
|||
this.#firstRun = false;
|
||||
}
|
||||
|
||||
const loaded = Object.values(this.models).filter((model) => model).length;
|
||||
if (loaded !== count) this.#emit('load');
|
||||
const current = Math.trunc(now() - timeStamp);
|
||||
if (current > (this.performance.load as number || 0)) this.performance.load = current;
|
||||
}
|
||||
|
||||
// emit event
|
||||
/** @hidden */
|
||||
#emit = (event: string) => this.events?.dispatchEvent(new Event(event));
|
||||
|
||||
// check if backend needs initialization if it changed
|
||||
/** @hidden */
|
||||
#checkBackend = async (force = false) => {
|
||||
|
|
@ -433,9 +456,9 @@ export class Human {
|
|||
|
||||
/** Main detection method
|
||||
* - Analyze configuration: {@link Config}
|
||||
* - Pre-process input: {@link Input}
|
||||
* - Pre-this.process input: {@link Input}
|
||||
* - Run inference for all configured models
|
||||
* - Process and return result: {@link Result}
|
||||
* - this.process and return result: {@link Result}
|
||||
*
|
||||
* @param input: Input
|
||||
* @param userConfig?: {@link Config}
|
||||
|
|
@ -468,34 +491,35 @@ export class Human {
|
|||
await this.load();
|
||||
|
||||
timeStamp = now();
|
||||
let process = image.process(input, this.config);
|
||||
this.process = image.process(input, this.config);
|
||||
this.performance.image = Math.trunc(now() - timeStamp);
|
||||
this.analyze('Get Image:');
|
||||
|
||||
// run segmentation preprocessing
|
||||
if (this.config.segmentation.enabled && process && process.tensor) {
|
||||
// run segmentation prethis.processing
|
||||
if (this.config.segmentation.enabled && this.process && this.process.tensor) {
|
||||
this.analyze('Start Segmentation:');
|
||||
this.state = 'run:segmentation';
|
||||
timeStamp = now();
|
||||
await segmentation.predict(process);
|
||||
await segmentation.predict(this.process);
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
|
||||
if (process.canvas) {
|
||||
if (this.process.canvas) {
|
||||
// replace input
|
||||
tf.dispose(process.tensor);
|
||||
process = image.process(process.canvas, this.config);
|
||||
tf.dispose(this.process.tensor);
|
||||
this.process = image.process(this.process.canvas, this.config);
|
||||
}
|
||||
this.analyze('End Segmentation:');
|
||||
}
|
||||
|
||||
if (!process || !process.tensor) {
|
||||
if (!this.process || !this.process.tensor) {
|
||||
log('could not convert input to tensor');
|
||||
resolve({ error: 'could not convert input to tensor' });
|
||||
return;
|
||||
}
|
||||
this.#emit('image');
|
||||
|
||||
timeStamp = now();
|
||||
this.config.skipFrame = await this.#skipFrame(process.tensor);
|
||||
this.config.skipFrame = await this.#skipFrame(this.process.tensor);
|
||||
if (!this.performance.frames) this.performance.frames = 0;
|
||||
if (!this.performance.cached) this.performance.cached = 0;
|
||||
(this.performance.frames as number)++;
|
||||
|
|
@ -512,12 +536,12 @@ export class Human {
|
|||
|
||||
// run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
|
||||
if (this.config.async) {
|
||||
faceRes = this.config.face.enabled ? face.detectFace(this, process.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? face.detectFace(this, this.process.tensor) : [];
|
||||
if (this.performance.face) delete this.performance.face;
|
||||
} else {
|
||||
this.state = 'run:face';
|
||||
timeStamp = now();
|
||||
faceRes = this.config.face.enabled ? await face.detectFace(this, process.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? await face.detectFace(this, this.process.tensor) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.face = elapsedTime;
|
||||
}
|
||||
|
|
@ -525,18 +549,18 @@ export class Human {
|
|||
// run body: can be posenet, blazepose, efficientpose, movenet
|
||||
this.analyze('Start Body:');
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.performance.body) delete this.performance.body;
|
||||
} else {
|
||||
this.state = 'run:body';
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.body = elapsedTime;
|
||||
}
|
||||
|
|
@ -545,12 +569,12 @@ export class Human {
|
|||
// run handpose
|
||||
this.analyze('Start Hand:');
|
||||
if (this.config.async) {
|
||||
handRes = this.config.hand.enabled ? handpose.predict(process.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? handpose.predict(this.process.tensor, this.config) : [];
|
||||
if (this.performance.hand) delete this.performance.hand;
|
||||
} else {
|
||||
this.state = 'run:hand';
|
||||
timeStamp = now();
|
||||
handRes = this.config.hand.enabled ? await handpose.predict(process.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? await handpose.predict(this.process.tensor, this.config) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.hand = elapsedTime;
|
||||
}
|
||||
|
|
@ -559,14 +583,14 @@ export class Human {
|
|||
// run nanodet
|
||||
this.analyze('Start Object:');
|
||||
if (this.config.async) {
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.performance.object) delete this.performance.object;
|
||||
} else {
|
||||
this.state = 'run:object';
|
||||
timeStamp = now();
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.object = elapsedTime;
|
||||
}
|
||||
|
|
@ -586,6 +610,7 @@ export class Human {
|
|||
|
||||
this.performance.total = Math.trunc(now() - timeStart);
|
||||
this.state = 'idle';
|
||||
const shape = this.process?.tensor?.shape || [];
|
||||
this.result = {
|
||||
face: faceRes as Face[],
|
||||
body: bodyRes as Body[],
|
||||
|
|
@ -593,15 +618,16 @@ export class Human {
|
|||
gesture: gestureRes,
|
||||
object: objectRes as Item[],
|
||||
performance: this.performance,
|
||||
canvas: process.canvas,
|
||||
canvas: this.process.canvas,
|
||||
timestamp: Date.now(),
|
||||
get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, process?.tensor?.shape); },
|
||||
get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); },
|
||||
};
|
||||
|
||||
// finally dispose input tensor
|
||||
tf.dispose(process.tensor);
|
||||
tf.dispose(this.process.tensor);
|
||||
|
||||
// log('Result:', result);
|
||||
this.#emit('detect');
|
||||
resolve(this.result);
|
||||
});
|
||||
}
|
||||
|
|
@ -700,6 +726,7 @@ export class Human {
|
|||
else res = await this.#warmupNode();
|
||||
const t1 = now();
|
||||
if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res);
|
||||
this.#emit('warmup');
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ export interface Result {
|
|||
/** global performance object with timing values for each operation */
|
||||
performance: Record<string, unknown>,
|
||||
/** optional processed canvas that can be used to draw input on screen */
|
||||
canvas?: OffscreenCanvas | HTMLCanvasElement,
|
||||
canvas?: OffscreenCanvas | HTMLCanvasElement | null,
|
||||
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
|
||||
readonly timestamp: number,
|
||||
/** getter property that returns unified persons object */
|
||||
|
|
|
|||
Loading…
Reference in New Issue