mirror of https://github.com/vladmandic/human
support for dynamic backend switching
Vladimir Mandic
parent
04fcbc7e6a
commit
ded141a161
34
src/human.ts
34
src/human.ts
|
|
@ -2,7 +2,7 @@
|
|||
* Human main module
|
||||
*/
|
||||
|
||||
import { log, now, mergeDeep, validate, wait } from './helpers';
|
||||
import { log, now, mergeDeep, validate } from './helpers';
|
||||
import { Config, defaults } from './config';
|
||||
import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
|
|
@ -252,7 +252,11 @@ export class Human {
|
|||
}
|
||||
|
||||
/** Reset configuration to default values */
|
||||
reset = () => this.config = JSON.parse(JSON.stringify(defaults));
|
||||
reset = () => {
|
||||
const currentBackend = this.config.backend; // save backend;
|
||||
this.config = JSON.parse(JSON.stringify(defaults));
|
||||
this.config.backend = currentBackend;
|
||||
}
|
||||
|
||||
/** Validate current configuration schema */
|
||||
validate = (userConfig?: Partial<Config>) => validate(defaults, userConfig || this.config);
|
||||
|
|
@ -314,12 +318,13 @@ export class Human {
|
|||
/** Explicit backend initialization
|
||||
* - Normally done implicitly during initial load phase
|
||||
* - Call to explictly register and initialize TFJS backend without any other operations
|
||||
* - Used in webworker environments where there can be multiple instances of Human and not all initialized
|
||||
* - Use when changing backend during runtime
|
||||
*
|
||||
* @return Promise<void>
|
||||
*/
|
||||
init() {
|
||||
backend.check(this);
|
||||
async init() {
|
||||
await backend.check(this, true);
|
||||
await this.tf.ready();
|
||||
env.set(this.env);
|
||||
}
|
||||
|
||||
|
|
@ -396,7 +401,7 @@ export class Human {
|
|||
*/
|
||||
async detect(input: Input, userConfig?: Partial<Config>): Promise<Result | Error> {
|
||||
// detection happens inside a promise
|
||||
if (this.config.yield) await wait(1);
|
||||
this.state = 'detect';
|
||||
return new Promise(async (resolve) => {
|
||||
this.state = 'config';
|
||||
let timeStamp;
|
||||
|
|
@ -421,8 +426,8 @@ export class Human {
|
|||
// load models if enabled
|
||||
await this.load();
|
||||
|
||||
if (this.config.yield) await wait(1);
|
||||
timeStamp = now();
|
||||
this.state = 'image';
|
||||
let img = image.process(input, this.config);
|
||||
this.process = img;
|
||||
this.performance.image = Math.trunc(now() - timeStamp);
|
||||
|
|
@ -431,7 +436,7 @@ export class Human {
|
|||
// run segmentation prethis.processing
|
||||
if (this.config.segmentation.enabled && this.process && img.tensor && img.canvas) {
|
||||
this.analyze('Start Segmentation:');
|
||||
this.state = 'run:segmentation';
|
||||
this.state = 'detect:segmentation';
|
||||
timeStamp = now();
|
||||
await segmentation.predict(img);
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
|
|
@ -468,7 +473,7 @@ export class Human {
|
|||
let objectRes: ObjectResult[] | Promise<ObjectResult[]> | never[] = [];
|
||||
|
||||
// run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
|
||||
this.state = 'run:face';
|
||||
this.state = 'detect:face';
|
||||
if (this.config.async) {
|
||||
faceRes = this.config.face.enabled ? face.detectFace(this, img.tensor) : [];
|
||||
if (this.performance.face) delete this.performance.face;
|
||||
|
|
@ -481,7 +486,7 @@ export class Human {
|
|||
|
||||
// run body: can be posenet, blazepose, efficientpose, movenet
|
||||
this.analyze('Start Body:');
|
||||
this.state = 'run:body';
|
||||
this.state = 'detect:body';
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(img.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(img.tensor, this.config) : [];
|
||||
|
|
@ -501,7 +506,7 @@ export class Human {
|
|||
|
||||
// run handpose
|
||||
this.analyze('Start Hand:');
|
||||
this.state = 'run:hand';
|
||||
this.state = 'detect:hand';
|
||||
if (this.config.async) {
|
||||
handRes = this.config.hand.enabled ? handpose.predict(img.tensor, this.config) : [];
|
||||
if (this.performance.hand) delete this.performance.hand;
|
||||
|
|
@ -515,7 +520,7 @@ export class Human {
|
|||
|
||||
// run nanodet
|
||||
this.analyze('Start Object:');
|
||||
this.state = 'run:object';
|
||||
this.state = 'detect:object';
|
||||
if (this.config.async) {
|
||||
if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(img.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(img.tensor, this.config) : [];
|
||||
|
|
@ -530,12 +535,11 @@ export class Human {
|
|||
this.analyze('End Object:');
|
||||
|
||||
// if async wait for results
|
||||
this.state = 'run:await';
|
||||
if (this.config.yield) await wait(1);
|
||||
this.state = 'detect:await';
|
||||
if (this.config.async) [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
|
||||
|
||||
// run gesture analysis last
|
||||
this.state = 'run:gesture';
|
||||
this.state = 'detect:gesture';
|
||||
let gestureRes: GestureResult[] = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import * as humangl from './humangl';
|
|||
import * as env from '../env';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
|
||||
export async function check(instance) {
|
||||
if (env.env.initial || (instance.config.backend && (instance.config.backend.length > 0) && (tf.getBackend() !== instance.config.backend))) {
|
||||
export async function check(instance, force = false) {
|
||||
instance.state = 'backend';
|
||||
if (force || env.env.initial || (instance.config.backend && (instance.config.backend.length > 0) && (tf.getBackend() !== instance.config.backend))) {
|
||||
const timeStamp = now();
|
||||
instance.state = 'backend';
|
||||
|
||||
if (instance.config.backend && instance.config.backend.length > 0) {
|
||||
// detect web worker
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ async function warmupBitmap(instance) {
|
|||
let res;
|
||||
switch (instance.config.warmup) {
|
||||
case 'face': blob = await b64toBlob(sample.face); break;
|
||||
case 'body':
|
||||
case 'full': blob = await b64toBlob(sample.body); break;
|
||||
default: blob = null;
|
||||
}
|
||||
|
|
@ -98,14 +99,17 @@ async function warmupNode(instance) {
|
|||
*/
|
||||
export async function warmup(instance, userConfig?: Partial<Config>): Promise<Result | { error }> {
|
||||
const t0 = now();
|
||||
instance.state = 'warmup';
|
||||
if (userConfig) instance.config = mergeDeep(instance.config, userConfig) as Config;
|
||||
if (!instance.config.warmup || instance.config.warmup === 'none') return { error: 'null' };
|
||||
let res;
|
||||
if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
|
||||
else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
|
||||
else res = await warmupNode(instance);
|
||||
const t1 = now();
|
||||
if (instance.config.debug) log('Warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
|
||||
instance.emit('warmup');
|
||||
return res;
|
||||
return new Promise(async (resolve) => {
|
||||
if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
|
||||
else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
|
||||
else res = await warmupNode(instance);
|
||||
const t1 = now();
|
||||
if (instance.config.debug) log('Warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
|
||||
instance.emit('warmup');
|
||||
resolve(res);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,28 +14,30 @@
|
|||
@font-face { font-family: 'Lato'; font-display: swap; font-style: normal; font-weight: 100; src: local('Lato'), url('../../assets/lato-light.woff2') }
|
||||
html { font-family: 'Lato', 'Segoe UI'; font-size: 14px; font-variant: small-caps; }
|
||||
body { margin: 0; background: black; color: white; }
|
||||
canvas { position: absolute; bottom: 10px; right: 10px; width: 256px; height: 256px; }
|
||||
pre { line-height: 150%; }
|
||||
.events { position: absolute; top: 10px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; }
|
||||
.state { position: absolute; top: 60px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; }
|
||||
.canvas { position: absolute; bottom: 10px; right: 10px; width: 256px; height: 256px; z-index: 99; }
|
||||
.pre { line-height: 150%; }
|
||||
.events { position: absolute; top: 10px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; z-index: 99; }
|
||||
.state { position: absolute; top: 60px; right: 10px; width: 12rem; height: 1.25rem; background-color: grey; padding: 8px; z-index: 99; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<pre id="log"></pre>
|
||||
<pre id="log" class="pre"></pre>
|
||||
<div id="events" class="events"></div>
|
||||
<div id="state" class="state"></div>
|
||||
<canvas id="canvas" class="canvas" width=256 height=256></canvas>
|
||||
<script type="module">
|
||||
import Human from '../dist/human.esm.js';
|
||||
|
||||
const config = {
|
||||
async: true,
|
||||
warmup: 'full',
|
||||
warmup: 'none',
|
||||
debug: true,
|
||||
cacheSensitivity: 0,
|
||||
object: { enabled: true },
|
||||
}
|
||||
|
||||
const backends = ['wasm', 'webgl', 'humangl'];
|
||||
// const backends = ['wasm', 'wasm'];
|
||||
// const backends = ['humangl'];
|
||||
|
||||
const start = performance.now();
|
||||
|
|
@ -50,9 +52,14 @@
|
|||
return line + '\n';
|
||||
}
|
||||
|
||||
let last = new Date();
|
||||
async function log(...msgs) {
|
||||
document.getElementById('log').innerHTML += str(...msgs);
|
||||
console.log(...msgs);
|
||||
const dt = new Date();
|
||||
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
|
||||
const elap = (dt - last).toString().padStart(5, '0');
|
||||
document.getElementById('log').innerHTML += ts + ' +' + elap + 'ms' + ' ' + str(...msgs);
|
||||
console.log(ts, elap, ...msgs);
|
||||
last = dt;
|
||||
}
|
||||
|
||||
async function image(url) {
|
||||
|
|
@ -69,6 +76,13 @@
|
|||
await waiting;
|
||||
}
|
||||
|
||||
function draw(canvas = null) {
|
||||
const c = document.getElementById('canvas');
|
||||
const ctx = c.getContext('2d');
|
||||
if (canvas) ctx.drawImage(canvas, 0, 0, c.width, c.height);
|
||||
else ctx.clearRect(0, 0, c.width, c.height);
|
||||
}
|
||||
|
||||
async function events(event) {
|
||||
document.getElementById('events').innerText = `${Math.round(performance.now() - start)}ms Event: ${event}`;
|
||||
}
|
||||
|
|
@ -77,66 +91,62 @@
|
|||
log('human tests');
|
||||
let res;
|
||||
let human = new Human(config);
|
||||
setInterval(() => { document.getElementById('state').innerText = `State: ${human.state}`; }, 10);
|
||||
human.events.addEventListener('warmup', () => events('warmup'));
|
||||
human.events.addEventListener('image', () => events('image'));
|
||||
human.events.addEventListener('detect', () => events('detect'));
|
||||
const timer = setInterval(() => { document.getElementById('state').innerText = `State: ${human.state}`; }, 10);
|
||||
log({ version: human.version });
|
||||
log({ env: human.env });
|
||||
log({ config: human.config });
|
||||
log({ tfjs: human.tf.version.tfjs, backend: human.config.backend });
|
||||
await human.load();
|
||||
const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) }));
|
||||
log({ models });
|
||||
for (const backend of backends) {
|
||||
log('');
|
||||
log();
|
||||
log('test start:', backend);
|
||||
human.config.backend = backend;
|
||||
human = new Human(config);
|
||||
human.events.addEventListener('warmup', () => events('warmup'));
|
||||
human.events.addEventListener('image', () => events('image'));
|
||||
human.events.addEventListener('detect', () => events('detect'));
|
||||
await human.load();
|
||||
human.env.offscreen = false;
|
||||
human.env.initial = false;
|
||||
await human.init();
|
||||
log({ tfjs: human.tf.version.tfjs, backend: human.tf.getBackend() });
|
||||
const models = Object.keys(human.models).map((model) => ({ name: model, loaded: (human.models[model] !== null) }));
|
||||
log({ models: { models }});
|
||||
log({ memory: human.tf.engine().state });
|
||||
log({ initialized: human.tf.getBackend() });
|
||||
log({ memory: human.tf.memory() });
|
||||
res = await human.validate();
|
||||
log({ validate: res });
|
||||
res = await human.warmup();
|
||||
log({ warmup: res });
|
||||
res = await human.warmup({ warmup: 'face'});
|
||||
draw(res.canvas);
|
||||
log({ warmup: 'face' });
|
||||
let img = await image('../../samples/ai-body.jpg');
|
||||
const input = await human.image(img);
|
||||
let node = document.body.appendChild(res.canvas);
|
||||
await wait(100);
|
||||
log({ input });
|
||||
log({ input: input.tensor.shape });
|
||||
draw(res.canvas);
|
||||
res = await human.detect(input.tensor);
|
||||
log({ detect: res});
|
||||
log({ detect: true });
|
||||
const interpolated = human.next();
|
||||
log({ interpolated });
|
||||
log({ interpolated: true });
|
||||
const persons = res.persons;
|
||||
log({ persons: { persons } });
|
||||
log({ persons: true });
|
||||
log({ summary: { persons: persons.length, face: res.face.length, body: res.body.length, hand: res.hand.length, object: res.object.length, gesture: res.gesture.length }});
|
||||
log({ performance: human.performance });
|
||||
human.tf.dispose(input.tensor);
|
||||
document.body.removeChild(node);
|
||||
await wait(100);
|
||||
draw();
|
||||
|
||||
img = await image('../../samples/ai-face.jpg');
|
||||
human.reset();
|
||||
human.config.backend = backend;
|
||||
for (const val of [0, 0.25, 0.5, 0.75, 10]) {
|
||||
human.performance = {};
|
||||
const t0 = performance.now();
|
||||
for (let i = 0; i < 10; i++) {
|
||||
res = await human.detect(img, { cacheSensitivity: val, filter: { pixelate: 5 * i } });
|
||||
node = document.body.appendChild(res.canvas);
|
||||
res = await human.detect(img, { cacheSensitivity: val, filter: { pixelate: 5 * i }, object: { enabled: false } });
|
||||
draw(res.canvas);
|
||||
}
|
||||
const t1 = performance.now();
|
||||
log({ benchmark: { time: Math.round((t1 - t0) / 10), cacheSensitivity: val }, performance: human.performance });
|
||||
await wait(100);
|
||||
await wait(10);
|
||||
}
|
||||
document.body.removeChild(node);
|
||||
draw();
|
||||
|
||||
log({ memory: human.tf.engine().state });
|
||||
log({ memory: human.tf.memory() });
|
||||
}
|
||||
log('');
|
||||
clearInterval(timer);
|
||||
log();
|
||||
log('tests complete');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -233,8 +233,8 @@ async function test(Human, inputConfig) {
|
|||
res1 = Math.round(100 * human.similarity(desc1, desc2));
|
||||
res2 = Math.round(100 * human.similarity(desc1, desc3));
|
||||
res3 = Math.round(100 * human.similarity(desc2, desc3));
|
||||
if (res1 !== 51 || res2 !== 49 || res3 !== 53) log('error', 'failed: face match ', res1, res2, res3);
|
||||
else log('state', 'passed: face match');
|
||||
if (res1 !== 51 || res2 !== 49 || res3 !== 53) log('error', 'failed: face similarity ', res1, res2, res3);
|
||||
else log('state', 'passed: face similarity');
|
||||
|
||||
// test face matching
|
||||
log('info', 'test face matching');
|
||||
|
|
@ -247,7 +247,7 @@ async function test(Human, inputConfig) {
|
|||
res1 = human.match(desc1, db);
|
||||
res2 = human.match(desc2, db);
|
||||
res3 = human.match(desc3, db);
|
||||
if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1);
|
||||
if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1, res2, res3);
|
||||
else log('state', 'passed: face match', { first: { name: res1.name, similarity: res1.similarity } }, { second: { name: res2.name, similarity: res2.similarity } }, { third: { name: res3.name, similarity: res3.similarity } });
|
||||
|
||||
// test object detection
|
||||
|
|
|
|||
Loading…
Reference in New Issue