start working on efficientpose

pull/92/head
Vladimir Mandic 2021-03-26 18:50:19 -04:00
parent d01ec955dd
commit f37be8efe5
27 changed files with 1437 additions and 1260 deletions

View File

@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/03/26 mandic00@live.com
### **1.2.5** 2021/03/25 mandic00@live.com
- fix broken exports

View File

@ -3,22 +3,20 @@ import Human from '../src/human';
import Menu from './menu.js';
import GLBench from './gl-bench.js';
const userConfig = { backend: 'webgl' }; // add any user configuration overrides
// const userConfig = { backend: 'webgl' }; // add any user configuration overrides
/*
const userConfig = {
backend: 'wasm',
backend: 'webgl',
async: false,
warmup: 'full',
videoOptimized: true,
filter: { enabled: true },
face: { enabled: true, mesh: { enabled: true }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
face: { enabled: false, mesh: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false, modelPath: '../models/blazepose.json' },
body: { enabled: true, modelPath: '../models/efficientpose.json' },
object: { enabled: false },
};
*/
const human = new Human(userConfig);

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

522
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

522
dist/human.js vendored

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

16
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

514
dist/tfjs.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -56,14 +56,14 @@
"@tensorflow/tfjs-layers": "^3.3.0",
"@tensorflow/tfjs-node": "^3.3.0",
"@tensorflow/tfjs-node-gpu": "^3.3.0",
"@types/node": "^14.14.35",
"@types/node": "^14.14.36",
"@typescript-eslint/eslint-plugin": "^4.19.0",
"@typescript-eslint/parser": "^4.19.0",
"@vladmandic/pilogger": "^0.2.15",
"chokidar": "^3.5.1",
"dayjs": "^1.10.4",
"esbuild": "^0.10.0",
"eslint": "^7.22.0",
"esbuild": "^0.10.1",
"eslint": "^7.23.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-json": "^2.1.2",

View File

@ -59,7 +59,7 @@ export class MediaPipeFaceMesh {
}
let faceModels = [null, null, null];
export async function load(config) {
export async function load(config): Promise<MediaPipeFaceMesh> {
// @ts-ignore
faceModels = await Promise.all([
(!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,

View File

@ -221,12 +221,14 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
}
if (drawOptions.drawLabels) {
ctx.font = drawOptions.font;
for (const pt of result[i].keypoints) {
ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
if (result[i].keypoints) {
for (const pt of result[i].keypoints) {
ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
}
}
}
if (drawOptions.drawPolygons) {
if (drawOptions.drawPolygons && result[i].keypoints) {
let part;
const points: any[] = [];
// torso

View File

@ -0,0 +1,97 @@
import { log } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
let model;
let last = { };
let skipped = Number.MAX_SAFE_INTEGER;
const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle'];
export async function load(config) {
if (!model) {
model = await tf.loadGraphModel(config.body.modelPath);
if (config.debug) log(`load model: ${config.body.modelPath.match(/\/(.*)\./)[1]}`);
}
return model;
}
// performs argmax and max functions on a 2d tensor
function max2d(inputs, minScore) {
const [width, height] = inputs.shape;
return tf.tidy(() => {
// modulus op implemented in tf
const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32')));
// combine all data
const reshaped = tf.reshape(inputs, [height * width]);
// get highest score
const score = tf.max(reshaped, 0).dataSync()[0];
if (score > minScore) {
// skip coordinate calculation is score is too low
const coords = tf.argMax(reshaped, 0);
const x = mod(coords, width).dataSync()[0];
const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0];
return [x, y, score];
}
return [0, 0, score];
});
}
export async function predict(image, config) {
if (!model) return null;
if ((skipped < config.body.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
skipped++;
return last;
}
if (config.videoOptimized) skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const enhance = tf.mul(resize, [255.0]);
tf.dispose(resize);
let resT;
if (!config.profile) {
if (config.body.enabled) resT = await model.predict(enhance);
} else {
const profileT = config.body.enabled ? await tf.profile(() => model.predict(enhance)) : {};
resT = profileT.result.clone();
profileT.result.dispose();
profile.run('body', profileT);
}
enhance.dispose();
if (resT) {
const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = [];
const squeeze = resT.squeeze();
tf.dispose(resT);
// body parts are basically just a stack of 2d tensors
const stack = squeeze.unstack(2);
tf.dispose(squeeze);
// process each unstacked tensor as a separate body part
for (let id = 0; id < stack.length; id++) {
// actual processing to get coordinates and score
const [x, y, score] = max2d(stack[id], config.body.scoreThreshold);
if (score > config.body.scoreThreshold) {
parts.push({
id,
score,
part: bodyParts[id],
positionRaw: {
xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1
yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1
},
position: {
x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size
y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), // y normalized to input image size
},
});
}
}
stack.forEach((s) => tf.dispose(s));
last = parts;
}
resolve(last);
});
}

View File

@ -55,7 +55,7 @@ export class HandPose {
}
}
export async function load(config) {
export async function load(config): Promise<HandPose> {
const [handDetectorModel, handPoseModel] = await Promise.all([
config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,

View File

@ -12,6 +12,7 @@ import * as embedding from './embedding/embedding';
import * as posenet from './posenet/posenet';
import * as handpose from './handpose/handpose';
import * as blazepose from './blazepose/blazepose';
import * as efficientpose from './efficientpose/efficientpose';
import * as nanodet from './nanodet/nanodet';
import * as gesture from './gesture/gesture';
import * as image from './image/image';
@ -62,9 +63,10 @@ export class Human {
};
// models
models: {
face: facemesh.MediaPipeFaceMesh | null,
face: facemesh.MediaPipeFaceMesh | Model | null,
posenet: posenet.PoseNet | null,
blazepose: Model | null,
efficientpose: Model | null,
handpose: handpose.HandPose | null,
iris: Model | null,
age: Model | null,
@ -108,6 +110,7 @@ export class Human {
face: null,
posenet: null,
blazepose: null,
efficientpose: null,
handpose: null,
iris: null,
age: null,
@ -206,9 +209,12 @@ export class Human {
this.models.gender,
this.models.emotion,
this.models.embedding,
// @ts-ignore
this.models.handpose,
// @ts-ignore false warning with latest @typescript-eslint
this.models.posenet,
this.models.blazepose,
this.models.efficientpose,
this.models.nanodet,
this.models.faceres,
] = await Promise.all([
@ -217,9 +223,10 @@ export class Human {
this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null),
this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null),
this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null),
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
this.models.handpose || (this.config.hand.enabled ? <Promise<handpose.HandPose>>handpose.load(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null),
this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null),
this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
]);
@ -232,6 +239,7 @@ export class Human {
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
if (this.config.body.enabled && !this.models.efficientpose && this.config.body.modelPath.includes('efficientpose')) this.models.efficientpose = await efficientpose.load(this.config);
if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
}
@ -363,13 +371,15 @@ export class Human {
this.analyze('Start Body:');
if (this.config.async) {
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
else bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
if (this.perf.body) delete this.perf.body;
} else {
this.state = 'run:body';
timeStamp = now();
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
else bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp);
if (current > 0) this.perf.body = current;
}

View File

@ -0,0 +1,61 @@
import { log } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';
/*
Prototype implementation for model processing
Must implement
- load()
- predict()
Must account for:
- image processing, tfjs profiling
*/
let model;
let last = { };
let skipped = Number.MAX_SAFE_INTEGER;
export async function load(config) {
if (!model) {
model = await tf.loadGraphModel(config.prototype.modelPath);
if (config.debug) log(`load model: ${config.prototype.modelPath.match(/\/(.*)\./)[1]}`);
}
return model;
}
export async function predict(image, config) {
if (!model) return null;
if ((skipped < config.prototype.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
skipped++;
return last;
}
if (config.videoOptimized) skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const enhance = tf.mul(resize, [255.0]);
tf.dispose(resize);
let resT;
if (!config.profile) {
if (config.prototype.enabled) resT = await model.predict(enhance);
} else {
const profileT = config.prototype.enabled ? await tf.profile(() => model.predict(enhance)) : {};
resT = profileT.result.clone();
profileT.result.dispose();
profile.run('prototype', profileT);
}
enhance.dispose();
let obj = {};
if (resT) {
const data = resT.dataSync();
obj = { data };
tf.dispose(resT);
}
last = obj;
resolve(obj);
});
}

File diff suppressed because one or more lines are too long

View File

@ -425,7 +425,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
<a name="models" class="tsd-anchor"></a>
<h3>models</h3>
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-type-declaration">
@ -437,6 +437,9 @@
<li class="tsd-parameter">
<h5>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
<li class="tsd-parameter">
<h5>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
<li class="tsd-parameter">
<h5>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
@ -444,7 +447,7 @@
<h5>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
<li class="tsd-parameter">
<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span></h5>
<h5>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span></h5>
</li>
<li class="tsd-parameter">
<h5>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>

View File

@ -0,0 +1,2 @@
export declare function load(config: any): Promise<any>;
export declare function predict(image: any, config: any): Promise<unknown>;

3
types/human.d.ts vendored
View File

@ -54,9 +54,10 @@ export declare class Human {
all: typeof draw.all;
};
models: {
face: facemesh.MediaPipeFaceMesh | null;
face: facemesh.MediaPipeFaceMesh | Model | null;
posenet: posenet.PoseNet | null;
blazepose: Model | null;
efficientpose: Model | null;
handpose: handpose.HandPose | null;
iris: Model | null;
age: Model | null;