ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added support for multiple instances of human

pull/293/head
Vladimir Mandic 2021-04-12 08:29:52 -04:00
parent da8307b306
commit e3b5dcb75e
15 changed files with 80 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,7 +1,7 @@
# @vladmandic/human
Version: **1.4.1**
Description: **Human: AI-powered 3D Face Detection, Face Description & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
License: **MIT** </LICENSE>
@ -9,8 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/04/09 mandic00@live.com
### **HEAD -> main** 2021/04/10 mandic00@live.com
- fix typedoc
- exception handling
### **1.4.1** 2021/04/09 mandic00@live.com

2
demo/facematch.js
View File

@ -1,3 +1,5 @@
// @ts-nocheck
import Human from '../dist/human.esm.js';
const userConfig = {

2
demo/index.js
View File

@ -1,3 +1,5 @@
// @ts-nocheck
/* global tf */
import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
// import Human from '../dist/human.esm-nobundle.js'; // this requires that tf is loaded manually and bundled before human can be used

6
package.json
View File

@ -67,13 +67,13 @@
"@vladmandic/pilogger": "^0.2.16",
"chokidar": "^3.5.1",
"dayjs": "^1.10.4",
"esbuild": "^0.11.6",
"eslint": "^7.23.0",
"esbuild": "^0.11.9",
"eslint": "^7.24.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-json": "^2.1.2",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^4.3.1",
"eslint-plugin-promise": "^5.1.0",
"rimraf": "^3.0.2",
"seedrandom": "^3.0.5",
"simple-git": "^2.37.0",

2
src/age/age.ts
View File

@ -11,7 +11,7 @@ export async function load(config) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.face.age.modelPath));
if (!model || !model.modelUrl) log('load model failed:', config.face.age.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

2
src/efficientpose/efficientpose.ts
View File

@ -13,7 +13,7 @@ export async function load(config) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

2
src/emotion/emotion.ts
View File

@ -15,7 +15,7 @@ export async function load(config) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath));
if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

2
src/faceres/faceres.ts
View File

@ -14,7 +14,7 @@ export async function load(config) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.face.description.modelPath));
if (!model || !model.modelUrl) log('load model failed:', config.face.description.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

2
src/gender/gender.ts
View File

@ -16,7 +16,7 @@ export async function load(config) {
alternative = model.inputs[0].shape[3] === 1;
if (!model || !model.modelUrl) log('load model failed:', config.face.gender.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

28
src/handpose/handpose.ts
View File

@ -53,20 +53,26 @@ export class HandPose {
}
}
let handDetectorModel;
let handPoseModel;
export async function load(config): Promise<HandPose> {
const [handDetectorModel, handPoseModel] = await Promise.all([
config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
]);
if (!handDetectorModel || !handPoseModel) {
[handDetectorModel, handPoseModel] = await Promise.all([
config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
]);
if (config.hand.enabled) {
if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
else if (config.debug) log('load model:', handDetectorModel.modelUrl);
if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
else if (config.debug) log('load model:', handPoseModel.modelUrl);
}
} else {
if (config.debug) log('cached model:', handDetectorModel.modelUrl);
if (config.debug) log('cached model:', handPoseModel.modelUrl);
}
const handDetector = new handdetector.HandDetector(handDetectorModel, handDetectorModel?.inputs[0].shape[2], anchors.anchors);
const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, handPoseModel?.inputs[0].shape[2]);
const handPose = new HandPose(handPipeline);
if (config.hand.enabled) {
if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath);
else if (config.debug) log('load model:', handDetectorModel.modelUrl);
if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath);
else if (config.debug) log('load model:', handPoseModel.modelUrl);
}
return handPose;
}

12
src/human.ts
View File

@ -343,6 +343,14 @@ export class Human {
if (this.config.scoped) this.tf.engine().startScope();
this.analyze('Start Scope:');
// disable video optimization for inputs of type image
let previousVideoOptimized;
if (input && this.config.videoOptimized && (input instanceof HTMLImageElement || input instanceof Image || input instanceof ImageData || input instanceof ImageBitmap || input instanceof tf.Tensor)) {
log('disabling video optimization');
previousVideoOptimized = this.config.videoOptimized;
this.config.videoOptimized = false;
}
timeStamp = now();
const process = image.process(input, this.config);
if (!process || !process.tensor) {
@ -427,6 +435,7 @@ export class Human {
if (this.config.scoped) this.tf.engine().endScope();
this.analyze('End Scope:');
// run gesture analysis last
let gestureRes: any[] = [];
if (this.config.gesture.enabled) {
timeStamp = now();
@ -435,6 +444,9 @@ export class Human {
else if (this.perf.gesture) delete this.perf.gesture;
}
// restore video optimizations if previously disabled
if (previousVideoOptimized) this.config.videoOptimized = previousVideoOptimized;
this.perf.total = Math.trunc(now() - timeStart);
this.state = 'idle';
const result = {

3
src/image/image.ts
View File

@ -28,7 +28,8 @@ export function process(input, config): { tensor: typeof tf.Tensor | null, canva
throw new Error('Human: Input type is not recognized');
}
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) tensor = tf.clone(input);
else throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
} else {
const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0));
const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0));

2
src/nanodet/nanodet.ts
View File

@ -17,7 +17,7 @@ export async function load(config) {
if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
}
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}

5
src/result.ts
View File

@ -40,7 +40,10 @@ export interface Result {
emotion: Array<{ score: number, emotion: string }>,
embedding: Array<number>,
iris: number,
angle: { roll: number, yaw: number, pitch: number },
rotation: {
angle: { roll: number, yaw: number, pitch: number },
matrix: Array<[number, number, number, number, number, number, number, number, number]>
}
}>,
/** Body results
*

31
test/test-node.js
View File

@ -30,11 +30,16 @@ const config = {
object: { enabled: true },
};
async function test() {
const human = new Human(config);
async function testInstance(human) {
if (human) log.state('passed: create human');
else log.error('failed: create human');
// if (!human.tf) human.tf = tf;
log.info('human version:', human.version);
log.info('tfjs version:', human.tf.version_core);
log.info('platform:', human.sysinfo.platform);
log.info('agent:', human.sysinfo.agent);
await human.load();
if (human.models) {
log.state('passed: load models');
@ -53,6 +58,28 @@ async function test() {
} else {
log.error('failed: warmup');
}
const random = tf.randomNormal([1, 1024, 1024, 3]);
const detect = await human.detect(random);
tf.dispose(random);
if (detect) {
log.state('passed: detect:', 'random');
log.data(' result: face:', detect.face.length, 'body:', detect.body.length, 'hand:', detect.hand.length, 'gesture:', detect.gesture.length, 'object:', detect.object.length);
log.data(' result: performance:', 'load:', detect.performance.load, 'total:', detect.performance.total);
} else {
log.error('failed: detect');
}
}
async function test() {
log.info('testing instance#1');
config.warmup = 'face';
const human1 = new Human(config);
await testInstance(human1);
log.info('testing instance#2');
config.warmup = 'body';
const human2 = new Human(config);
await testInstance(human2);
}
test();