mirror of https://github.com/vladmandic/human
start working on efficientpose
Vladimir Mandic
parent
1dbeb93726
commit
1dd860d112
|
|
@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/03/26 mandic00@live.com
|
||||
|
||||
|
||||
### **1.2.5** 2021/03/25 mandic00@live.com
|
||||
|
||||
- fix broken exports
|
||||
|
|
|
|||
|
|
@ -3,22 +3,20 @@ import Human from '../src/human';
|
|||
import Menu from './menu.js';
|
||||
import GLBench from './gl-bench.js';
|
||||
|
||||
const userConfig = { backend: 'webgl' }; // add any user configuration overrides
|
||||
// const userConfig = { backend: 'webgl' }; // add any user configuration overrides
|
||||
|
||||
/*
|
||||
const userConfig = {
|
||||
backend: 'wasm',
|
||||
backend: 'webgl',
|
||||
async: false,
|
||||
warmup: 'full',
|
||||
videoOptimized: true,
|
||||
filter: { enabled: true },
|
||||
face: { enabled: true, mesh: { enabled: true }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
|
||||
face: { enabled: false, mesh: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: false, modelPath: '../models/blazepose.json' },
|
||||
body: { enabled: true, modelPath: '../models/efficientpose.json' },
|
||||
object: { enabled: false },
|
||||
};
|
||||
*/
|
||||
|
||||
const human = new Human(userConfig);
|
||||
|
||||
|
|
|
|||
|
|
@ -56,14 +56,14 @@
|
|||
"@tensorflow/tfjs-layers": "^3.3.0",
|
||||
"@tensorflow/tfjs-node": "^3.3.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.3.0",
|
||||
"@types/node": "^14.14.35",
|
||||
"@types/node": "^14.14.36",
|
||||
"@typescript-eslint/eslint-plugin": "^4.19.0",
|
||||
"@typescript-eslint/parser": "^4.19.0",
|
||||
"@vladmandic/pilogger": "^0.2.15",
|
||||
"chokidar": "^3.5.1",
|
||||
"dayjs": "^1.10.4",
|
||||
"esbuild": "^0.10.0",
|
||||
"eslint": "^7.22.0",
|
||||
"esbuild": "^0.10.1",
|
||||
"eslint": "^7.23.0",
|
||||
"eslint-config-airbnb-base": "^14.2.1",
|
||||
"eslint-plugin-import": "^2.22.1",
|
||||
"eslint-plugin-json": "^2.1.2",
|
||||
|
|
|
|||
|
|
@ -221,12 +221,14 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
}
|
||||
if (drawOptions.drawLabels) {
|
||||
ctx.font = drawOptions.font;
|
||||
for (const pt of result[i].keypoints) {
|
||||
ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
|
||||
ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
|
||||
if (result[i].keypoints) {
|
||||
for (const pt of result[i].keypoints) {
|
||||
ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
|
||||
ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (drawOptions.drawPolygons) {
|
||||
if (drawOptions.drawPolygons && result[i].keypoints) {
|
||||
let part;
|
||||
const points: any[] = [];
|
||||
// torso
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
import { log } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as profile from '../profile';
|
||||
|
||||
let model;
|
||||
let last = { };
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle'];
|
||||
|
||||
export async function load(config) {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(config.body.modelPath);
|
||||
if (config.debug) log(`load model: ${config.body.modelPath.match(/\/(.*)\./)[1]}`);
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
// performs argmax and max functions on a 2d tensor
|
||||
function max2d(inputs, minScore) {
|
||||
const [width, height] = inputs.shape;
|
||||
return tf.tidy(() => {
|
||||
// modulus op implemented in tf
|
||||
const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32')));
|
||||
// combine all data
|
||||
const reshaped = tf.reshape(inputs, [height * width]);
|
||||
// get highest score
|
||||
const score = tf.max(reshaped, 0).dataSync()[0];
|
||||
if (score > minScore) {
|
||||
// skip coordinate calculation is score is too low
|
||||
const coords = tf.argMax(reshaped, 0);
|
||||
const x = mod(coords, width).dataSync()[0];
|
||||
const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0];
|
||||
return [x, y, score];
|
||||
}
|
||||
return [0, 0, score];
|
||||
});
|
||||
}
|
||||
|
||||
export async function predict(image, config) {
|
||||
if (!model) return null;
|
||||
if ((skipped < config.body.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) {
|
||||
skipped++;
|
||||
return last;
|
||||
}
|
||||
if (config.videoOptimized) skipped = 0;
|
||||
else skipped = Number.MAX_SAFE_INTEGER;
|
||||
return new Promise(async (resolve) => {
|
||||
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const enhance = tf.mul(resize, [255.0]);
|
||||
tf.dispose(resize);
|
||||
|
||||
let resT;
|
||||
|
||||
if (!config.profile) {
|
||||
if (config.body.enabled) resT = await model.predict(enhance);
|
||||
} else {
|
||||
const profileT = config.body.enabled ? await tf.profile(() => model.predict(enhance)) : {};
|
||||
resT = profileT.result.clone();
|
||||
profileT.result.dispose();
|
||||
profile.run('body', profileT);
|
||||
}
|
||||
enhance.dispose();
|
||||
|
||||
if (resT) {
|
||||
const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = [];
|
||||
const squeeze = resT.squeeze();
|
||||
tf.dispose(resT);
|
||||
// body parts are basically just a stack of 2d tensors
|
||||
const stack = squeeze.unstack(2);
|
||||
tf.dispose(squeeze);
|
||||
// process each unstacked tensor as a separate body part
|
||||
for (let id = 0; id < stack.length; id++) {
|
||||
// actual processing to get coordinates and score
|
||||
const [x, y, score] = max2d(stack[id], config.body.scoreThreshold);
|
||||
if (score > config.body.scoreThreshold) {
|
||||
parts.push({
|
||||
id,
|
||||
score,
|
||||
part: bodyParts[id],
|
||||
positionRaw: {
|
||||
xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1
|
||||
yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1
|
||||
},
|
||||
position: {
|
||||
x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size
|
||||
y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), // y normalized to input image size
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
stack.forEach((s) => tf.dispose(s));
|
||||
last = parts;
|
||||
}
|
||||
resolve(last);
|
||||
});
|
||||
}
|
||||
|
|
@ -55,7 +55,7 @@ export class HandPose {
|
|||
}
|
||||
}
|
||||
|
||||
export async function load(config) {
|
||||
export async function load(config): Promise<HandPose> {
|
||||
const [handDetectorModel, handPoseModel] = await Promise.all([
|
||||
config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
|
||||
config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
|
||||
|
|
|
|||
20
src/human.ts
20
src/human.ts
|
|
@ -12,6 +12,7 @@ import * as embedding from './embedding/embedding';
|
|||
import * as posenet from './posenet/posenet';
|
||||
import * as handpose from './handpose/handpose';
|
||||
import * as blazepose from './blazepose/blazepose';
|
||||
import * as efficientpose from './efficientpose/efficientpose';
|
||||
import * as nanodet from './nanodet/nanodet';
|
||||
import * as gesture from './gesture/gesture';
|
||||
import * as image from './image/image';
|
||||
|
|
@ -62,9 +63,10 @@ export class Human {
|
|||
};
|
||||
// models
|
||||
models: {
|
||||
face: facemesh.MediaPipeFaceMesh | null,
|
||||
face: facemesh.MediaPipeFaceMesh | Model | null,
|
||||
posenet: posenet.PoseNet | null,
|
||||
blazepose: Model | null,
|
||||
efficientpose: Model | null,
|
||||
handpose: handpose.HandPose | null,
|
||||
iris: Model | null,
|
||||
age: Model | null,
|
||||
|
|
@ -108,6 +110,7 @@ export class Human {
|
|||
face: null,
|
||||
posenet: null,
|
||||
blazepose: null,
|
||||
efficientpose: null,
|
||||
handpose: null,
|
||||
iris: null,
|
||||
age: null,
|
||||
|
|
@ -206,9 +209,12 @@ export class Human {
|
|||
this.models.gender,
|
||||
this.models.emotion,
|
||||
this.models.embedding,
|
||||
// @ts-ignore
|
||||
this.models.handpose,
|
||||
// @ts-ignore false warning with latest @typescript-eslint
|
||||
this.models.posenet,
|
||||
this.models.blazepose,
|
||||
this.models.efficientpose,
|
||||
this.models.nanodet,
|
||||
this.models.faceres,
|
||||
] = await Promise.all([
|
||||
|
|
@ -217,9 +223,10 @@ export class Human {
|
|||
this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null),
|
||||
this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null),
|
||||
this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null),
|
||||
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
|
||||
this.models.handpose || (this.config.hand.enabled ? <Promise<handpose.HandPose>>handpose.load(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
|
||||
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
|
||||
this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null),
|
||||
this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
|
||||
]);
|
||||
|
|
@ -232,6 +239,7 @@ export class Human {
|
|||
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
|
||||
if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
|
||||
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
|
||||
if (this.config.body.enabled && !this.models.efficientpose && this.config.body.modelPath.includes('efficientpose')) this.models.efficientpose = await efficientpose.load(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config);
|
||||
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
|
||||
}
|
||||
|
|
@ -363,13 +371,15 @@ export class Human {
|
|||
this.analyze('Start Body:');
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
else bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
|
||||
if (this.perf.body) delete this.perf.body;
|
||||
} else {
|
||||
this.state = 'run:body';
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
else bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0) this.perf.body = current;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue