added experimental movenet-lightning and removed blazepose from default dist

pull/134/head
Vladimir Mandic 2021-05-29 09:20:01 -04:00
parent b5033e5633
commit cd6da52bdb
32 changed files with 2660 additions and 2192 deletions

View File

@ -11,9 +11,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/05/28 mandic00@live.com ### **HEAD -> main** 2021/05/28 mandic00@live.com
- added experimental face.rotation.gaze
### **origin/main** 2021/05/27 mandic00@live.com - fix and optimize for mobile platform
- lock typescript to 4.2 due to typedoc incompatibility with 4.3
### **1.9.4** 2021/05/27 mandic00@live.com ### **1.9.4** 2021/05/27 mandic00@live.com

View File

@ -218,7 +218,7 @@ Default models in Human library are:
- **Body Analysis**: PoseNet (AtomicBits version) - **Body Analysis**: PoseNet (AtomicBits version)
Note that alternative models are provided and can be enabled via configuration Note that alternative models are provided and can be enabled via configuration
For example, `PoseNet` model can be switched for `BlazePose` model depending on the use case For example, `PoseNet` model can be switched for `BlazePose`, `EfficientPose` or `MoveNet` model depending on the use case
For more info, see [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration) and [**List of Models**](https://github.com/vladmandic/human/wiki/Models) For more info, see [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration) and [**List of Models**](https://github.com/vladmandic/human/wiki/Models)

View File

@ -13,8 +13,6 @@ N/A
- InsightFace - InsightFace
RetinaFace detector and ArcFace recognition RetinaFace detector and ArcFace recognition
<https://github.com/deepinsight/insightface> <https://github.com/deepinsight/insightface>
- Blazepose
Needs detector before running pose to center the image
## In Progress ## In Progress

View File

@ -16,14 +16,15 @@ let human;
const userConfig = { const userConfig = {
warmup: 'none', warmup: 'none',
/*
backend: 'webgl', backend: 'webgl',
// async: false, async: false,
// cacheSensitivity: 0, cacheSensitivity: 0,
filter: { filter: {
enabled: false, enabled: false,
flip: false, flip: false,
}, },
face: { enabled: true, face: { enabled: false,
detector: { return: true }, detector: { return: true },
mesh: { enabled: true }, mesh: { enabled: true },
iris: { enabled: true }, iris: { enabled: true },
@ -31,10 +32,12 @@ const userConfig = {
emotion: { enabled: false }, emotion: { enabled: false },
}, },
hand: { enabled: false }, hand: { enabled: false },
body: { enabled: false, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'posenet.json' },
body: { enabled: true, modelPath: 'movenet-lightning.json' },
// body: { enabled: true, modelPath: 'blazepose.json' }, // body: { enabled: true, modelPath: 'blazepose.json' },
object: { enabled: false }, object: { enabled: false },
gesture: { enabled: true }, gesture: { enabled: true },
*/
}; };
const drawOptions = { const drawOptions = {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

628
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

628
dist/human.js vendored

File diff suppressed because one or more lines are too long

780
dist/human.node-gpu.js vendored

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

780
dist/human.node.js vendored

File diff suppressed because it is too large Load Diff

Binary file not shown.

File diff suppressed because one or more lines are too long

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@ -1,17 +1,17 @@
2021-05-28 15:51:39 INFO:  @vladmandic/human version 1.9.4 2021-05-29 09:19:14 INFO:  @vladmandic/human version 1.9.4
2021-05-28 15:51:39 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 2021-05-29 09:19:14 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-28 15:51:39 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 2021-05-29 09:19:14 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-28 15:51:39 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"} 2021-05-29 09:19:14 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-28 15:51:39 STATE: Build for: node type: node: {"imports":37,"importBytes":434116,"outputBytes":387673,"outputFiles":"dist/human.node.js"} 2021-05-29 09:19:14 STATE: Build for: node type: node: {"imports":39,"importBytes":443616,"outputBytes":396012,"outputFiles":"dist/human.node.js"}
2021-05-28 15:51:39 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"} 2021-05-29 09:19:14 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-28 15:51:39 STATE: Build for: nodeGPU type: node: {"imports":37,"importBytes":434124,"outputBytes":387677,"outputFiles":"dist/human.node-gpu.js"} 2021-05-29 09:19:14 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":443624,"outputBytes":396016,"outputFiles":"dist/human.node-gpu.js"}
2021-05-28 15:51:39 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"} 2021-05-29 09:19:14 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-28 15:51:39 STATE: Build for: nodeWASM type: node: {"imports":37,"importBytes":434191,"outputBytes":387749,"outputFiles":"dist/human.node-wasm.js"} 2021-05-29 09:19:14 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":443691,"outputBytes":396088,"outputFiles":"dist/human.node-wasm.js"}
2021-05-28 15:51:40 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"} 2021-05-29 09:19:14 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-28 15:51:40 STATE: Build for: browserNoBundle type: esm: {"imports":37,"importBytes":434218,"outputBytes":237474,"outputFiles":"dist/human.esm-nobundle.js"} 2021-05-29 09:19:15 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":443718,"outputBytes":242135,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-28 15:51:40 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111414,"outputFiles":"dist/tfjs.esm.js"} 2021-05-29 09:19:15 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111414,"outputFiles":"dist/tfjs.esm.js"}
2021-05-28 15:51:41 STATE: Build for: browserBundle type: iife: {"imports":37,"importBytes":1544238,"outputBytes":1345362,"outputFiles":"dist/human.js"} 2021-05-29 09:19:16 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1553738,"outputBytes":1349942,"outputFiles":"dist/human.js"}
2021-05-28 15:51:41 STATE: Build for: browserBundle type: esm: {"imports":37,"importBytes":1544238,"outputBytes":1345354,"outputFiles":"dist/human.esm.js"} 2021-05-29 09:19:16 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1553738,"outputBytes":1349934,"outputFiles":"dist/human.esm.js"}
2021-05-28 15:51:41 INFO:  Generate types: ["src/human.ts"] 2021-05-29 09:19:16 INFO:  Generate types: ["src/human.ts"]
2021-05-28 15:51:46 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 2021-05-29 09:19:22 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-28 15:51:46 INFO:  Generate TypeDocs: ["src/human.ts"] 2021-05-29 09:19:22 INFO:  Generate TypeDocs: ["src/human.ts"]

View File

@ -52,7 +52,7 @@ export async function predict(image, config): Promise<Body[]> {
}); });
} }
const x = keypoints.map((a) => a.position.x); const x = keypoints.map((a) => a.position.x);
const y = keypoints.map((a) => a.position.x); const y = keypoints.map((a) => a.position.y);
const box: [number, number, number, number] = [ const box: [number, number, number, number] = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),

View File

@ -288,11 +288,11 @@ const config: Config = {
body: { body: {
enabled: true, enabled: true,
modelPath: 'posenet.json', // body model, can be absolute path or relative to modelBasePath modelPath: 'posenet.json', // body model, can be absolute path or relative to modelBasePath
// can be 'posenet' or 'blazepose' // can be 'posenet', 'blazepose', 'efficientpose', 'movenet'
maxDetected: 1, // maximum number of people detected in the input maxDetected: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose // only valid for posenet as other models detects single pose
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
}, },
hand: { hand: {

View File

@ -40,6 +40,7 @@ export interface DrawOptions {
drawLabels: boolean, drawLabels: boolean,
drawBoxes: boolean, drawBoxes: boolean,
drawPolygons: boolean, drawPolygons: boolean,
drawGaze: boolean,
fillPolygons: boolean, fillPolygons: boolean,
useDepth: boolean, useDepth: boolean,
useCurves: boolean, useCurves: boolean,
@ -60,6 +61,7 @@ export const options: DrawOptions = {
drawLabels: <boolean>true, drawLabels: <boolean>true,
drawBoxes: <boolean>true, drawBoxes: <boolean>true,
drawPolygons: <boolean>true, drawPolygons: <boolean>true,
drawGaze: <boolean>false,
fillPolygons: <boolean>false, fillPolygons: <boolean>false,
useDepth: <boolean>true, useDepth: <boolean>true,
useCurves: <boolean>false, useCurves: <boolean>false,
@ -238,7 +240,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.fill(); ctx.fill();
} }
} }
if (f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) { if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) {
const leftGaze = [ const leftGaze = [
f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]), f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]),
f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]), f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]),

View File

@ -99,20 +99,20 @@ export async function predict(image, config): Promise<Body[]> {
} }
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x); const x = keypoints.map((a) => a.position.x);
const y = keypoints.map((a) => a.position.x); const y = keypoints.map((a) => a.position.y);
box = [ box = [
Math.min(...x), Math.min(...x),
Math.min(...y), Math.min(...y),
Math.max(...x) - Math.min(...x), Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...x), Math.max(...y) - Math.min(...y),
]; ];
const xRaw = keypoints.map((a) => a.positionRaw.x); const xRaw = keypoints.map((a) => a.positionRaw.x);
const yRaw = keypoints.map((a) => a.positionRaw.x); const yRaw = keypoints.map((a) => a.positionRaw.y);
boxRaw = [ boxRaw = [
Math.min(...xRaw), Math.min(...xRaw),
Math.min(...yRaw), Math.min(...yRaw),
Math.max(...xRaw) - Math.min(...xRaw), Math.max(...xRaw) - Math.min(...xRaw),
Math.max(...yRaw) - Math.min(...xRaw), Math.max(...yRaw) - Math.min(...yRaw),
]; ];
resolve([{ id: 0, score, box, boxRaw, keypoints }]); resolve([{ id: 0, score, box, boxRaw, keypoints }]);
}); });

View File

@ -15,6 +15,8 @@ import * as emotion from './emotion/emotion';
import * as posenet from './posenet/posenet'; import * as posenet from './posenet/posenet';
import * as handpose from './handpose/handpose'; import * as handpose from './handpose/handpose';
import * as blazepose from './blazepose/blazepose'; import * as blazepose from './blazepose/blazepose';
import * as efficientpose from './efficientpose/efficientpose';
import * as movenet from './movenet/movenet';
import * as nanodet from './object/nanodet'; import * as nanodet from './object/nanodet';
import * as centernet from './object/centernet'; import * as centernet from './object/centernet';
import * as gesture from './gesture/gesture'; import * as gesture from './gesture/gesture';
@ -91,6 +93,7 @@ export class Human {
posenet: Model | null, posenet: Model | null,
blazepose: Model | null, blazepose: Model | null,
efficientpose: Model | null, efficientpose: Model | null,
movenet: Model | null,
handpose: [Model, Model] | null, handpose: [Model, Model] | null,
iris: Model | null, iris: Model | null,
age: Model | null, age: Model | null,
@ -105,7 +108,7 @@ export class Human {
classes: { classes: {
facemesh: typeof facemesh; facemesh: typeof facemesh;
emotion: typeof emotion; emotion: typeof emotion;
body: typeof posenet | typeof blazepose; body: typeof posenet | typeof blazepose | typeof movenet;
hand: typeof handpose; hand: typeof handpose;
nanodet: typeof nanodet; nanodet: typeof nanodet;
centernet: typeof centernet; centernet: typeof centernet;
@ -150,6 +153,7 @@ export class Human {
posenet: null, posenet: null,
blazepose: null, blazepose: null,
efficientpose: null, efficientpose: null,
movenet: null,
handpose: null, handpose: null,
iris: null, iris: null,
age: null, age: null,
@ -262,6 +266,8 @@ export class Human {
this.models.handpose, this.models.handpose,
this.models.posenet, this.models.posenet,
this.models.blazepose, this.models.blazepose,
this.models.efficientpose,
this.models.movenet,
this.models.nanodet, this.models.nanodet,
this.models.centernet, this.models.centernet,
this.models.faceres, this.models.faceres,
@ -271,6 +277,8 @@ export class Human {
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null), this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null),
this.models.movenet || (this.config.body.enabled && this.config.body.modelPath.includes('movenet') ? movenet.load(this.config) : null),
this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes('nanodet') ? nanodet.load(this.config) : null), this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes('nanodet') ? nanodet.load(this.config) : null),
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes('centernet') ? centernet.load(this.config) : null), this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes('centernet') ? centernet.load(this.config) : null),
this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null), this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
@ -281,6 +289,8 @@ export class Human {
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config); if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config); if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config); if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
if (this.config.body.enabled && !this.models.efficientpose && this.config.body.modelPath.includes('efficientpose')) this.models.efficientpose = await blazepose.load(this.config);
if (this.config.body.enabled && !this.models.movenet && this.config.body.modelPath.includes('movenet')) this.models.movenet = await movenet.load(this.config);
if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes('nanodet')) this.models.nanodet = await nanodet.load(this.config); if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes('nanodet')) this.models.nanodet = await nanodet.load(this.config);
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes('centernet')) this.models.centernet = await centernet.load(this.config); if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes('centernet')) this.models.centernet = await centernet.load(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config); if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
@ -474,17 +484,21 @@ export class Human {
if (elapsedTime > 0) this.perf.face = elapsedTime; if (elapsedTime > 0) this.perf.face = elapsedTime;
} }
// run body: can be posenet or blazepose // run body: can be posenet, blazepose, efficientpose, movenet
this.analyze('Start Body:'); this.analyze('Start Body:');
if (this.config.async) { if (this.config.async) {
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : []; if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : []; else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(process.tensor, this.config) : [];
if (this.perf.body) delete this.perf.body; if (this.perf.body) delete this.perf.body;
} else { } else {
this.state = 'run:body'; this.state = 'run:body';
timeStamp = now(); timeStamp = now();
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : []; if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : []; else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(process.tensor, this.config) : [];
elapsedTime = Math.trunc(now() - timeStamp); elapsedTime = Math.trunc(now() - timeStamp);
if (elapsedTime > 0) this.perf.body = elapsedTime; if (elapsedTime > 0) this.perf.body = elapsedTime;
} }

92
src/movenet/movenet.ts Normal file
View File

@ -0,0 +1,92 @@
/**
* EfficientPose Module
*/
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import { Body } from '../result';
import { GraphModel } from '../tfjs/types';
let model: GraphModel;
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } };
const keypoints: Array<Keypoints> = [];
let box: [number, number, number, number] = [0, 0, 0, 0];
let boxRaw: [number, number, number, number] = [0, 0, 0, 0];
let score = 0;
let skipped = Number.MAX_SAFE_INTEGER;
const bodyParts = ['nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftShoulder', 'rightShoulder', 'leftElbow', 'rightElbow', 'leftWrist', 'rightWrist', 'leftHip', 'rightHip', 'leftKnee', 'rightKnee', 'leftAnkle', 'rightAnkle'];
export async function load(config) {
if (!model) {
// @ts-ignore type mismatch on GraphModel
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath));
if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath);
else if (config.debug) log('load model:', model['modelUrl']);
} else if (config.debug) log('cached model:', model['modelUrl']);
return model;
}
export async function predict(image, config): Promise<Body[]> {
if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
skipped++;
return [{ id: 0, score, box, boxRaw, keypoints }];
}
skipped = 0;
return new Promise(async (resolve) => {
const tensor = tf.tidy(() => {
if (!model.inputs[0].shape) return null;
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const cast = tf.cast(resize, 'int32');
return cast;
});
let resT;
if (config.body.enabled) resT = await model.predict(tensor);
tensor.dispose();
if (resT) {
keypoints.length = 0;
const res = resT.arraySync();
tf.dispose(resT);
const kpt = res[0][0];
for (let id = 0; id < kpt.length; id++) {
score = kpt[id][2];
if (score > config.body.minConfidence) {
keypoints.push({
score: Math.round(100 * score) / 100,
part: bodyParts[id],
positionRaw: { // normalized to 0..1
x: kpt[id][1],
y: kpt[id][0],
},
position: { // normalized to input image size
x: Math.round(image.shape[2] * kpt[id][1]),
y: Math.round(image.shape[1] * kpt[id][0]),
},
});
}
}
}
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x);
const y = keypoints.map((a) => a.position.y);
box = [
Math.min(...x),
Math.min(...y),
Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y),
];
const xRaw = keypoints.map((a) => a.positionRaw.x);
const yRaw = keypoints.map((a) => a.positionRaw.y);
boxRaw = [
Math.min(...xRaw),
Math.min(...yRaw),
Math.max(...xRaw) - Math.min(...xRaw),
Math.max(...yRaw) - Math.min(...yRaw),
];
resolve([{ id: 0, score, box, boxRaw, keypoints }]);
});
}

View File

@ -141,8 +141,8 @@ async function test(Human, inputConfig) {
log('info', 'test body variants'); log('info', 'test body variants');
config.body = { modelPath: 'posenet.json', enabled: true }; config.body = { modelPath: 'posenet.json', enabled: true };
await testDetect(human, 'assets/human-sample-body.jpg', 'posenet'); await testDetect(human, 'assets/human-sample-body.jpg', 'posenet');
config.body = { modelPath: 'blazepose.json', enabled: true }; config.body = { modelPath: 'movenet-lightning.json', enabled: true };
await testDetect(human, 'assets/human-sample-body.jpg', 'blazepose'); await testDetect(human, 'assets/human-sample-body.jpg', 'movenet');
await testDetect(human, null, 'default'); await testDetect(human, null, 'default');
log('info', 'test: first instance'); log('info', 'test: first instance');

File diff suppressed because one or more lines are too long

View File

@ -161,7 +161,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
<a name="classes" class="tsd-anchor"></a> <a name="classes" class="tsd-anchor"></a>
<h3>classes</h3> <h3>classes</h3>
<div class="tsd-signature tsd-kind-icon">classes<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>body<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>facemesh<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>hand<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">classes<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>body<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>facemesh<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>hand<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
@ -173,7 +173,7 @@
<h4>Type declaration</h4> <h4>Type declaration</h4>
<ul class="tsd-parameters"> <ul class="tsd-parameters">
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>body<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span></h5> <h5>body<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">__module</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span></h5> <h5>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">__module</span></h5>
@ -448,7 +448,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
<a name="models" class="tsd-anchor"></a> <a name="models" class="tsd-anchor"></a>
<h3>models</h3> <h3>models</h3>
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>centernet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>movenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-comment tsd-typography"> <div class="tsd-comment tsd-typography">
@ -492,6 +492,9 @@
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span></h5> <h5>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span></h5>
</li> </li>
<li class="tsd-parameter">
<h5>movenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span></h5>
</li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span></h5> <h5>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">unknown</span></h5>
</li> </li>

View File

@ -107,6 +107,7 @@
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#bufferedoutput" class="tsd-kind-icon">buffered<wbr>Output</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#color" class="tsd-kind-icon">color</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#color" class="tsd-kind-icon">color</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawgaze" class="tsd-kind-icon">draw<wbr>Gaze</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawpoints" class="tsd-kind-icon">draw<wbr>Points</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawpoints" class="tsd-kind-icon">draw<wbr>Points</a></li>
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawpolygons" class="tsd-kind-icon">draw<wbr>Polygons</a></li> <li class="tsd-kind-property tsd-parent-kind-interface"><a href="drawoptions.html#drawpolygons" class="tsd-kind-icon">draw<wbr>Polygons</a></li>
@ -155,6 +156,13 @@
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
</section> </section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="drawgaze" class="tsd-anchor"></a>
<h3>draw<wbr>Gaze</h3>
<div class="tsd-signature tsd-kind-icon">draw<wbr>Gaze<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">boolean</span></div>
<aside class="tsd-sources">
</aside>
</section>
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="drawlabels" class="tsd-anchor"></a> <a name="drawlabels" class="tsd-anchor"></a>
<h3>draw<wbr>Labels</h3> <h3>draw<wbr>Labels</h3>
@ -287,6 +295,9 @@
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a> <a href="drawoptions.html#drawboxes" class="tsd-kind-icon">draw<wbr>Boxes</a>
</li> </li>
<li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#drawgaze" class="tsd-kind-icon">draw<wbr>Gaze</a>
</li>
<li class=" tsd-kind-property tsd-parent-kind-interface"> <li class=" tsd-kind-property tsd-parent-kind-interface">
<a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a> <a href="drawoptions.html#drawlabels" class="tsd-kind-icon">draw<wbr>Labels</a>
</li> </li>

View File

@ -36,6 +36,7 @@ export interface DrawOptions {
drawLabels: boolean; drawLabels: boolean;
drawBoxes: boolean; drawBoxes: boolean;
drawPolygons: boolean; drawPolygons: boolean;
drawGaze: boolean;
fillPolygons: boolean; fillPolygons: boolean;
useDepth: boolean; useDepth: boolean;
useCurves: boolean; useCurves: boolean;

View File

@ -0,0 +1,6 @@
/**
* EfficientPose Module
*/
import { Body } from '../result';
export declare function load(config: any): Promise<any>;
export declare function predict(image: any, config: any): Promise<Body[]>;

4
types/human.d.ts vendored
View File

@ -10,6 +10,7 @@ import * as emotion from './emotion/emotion';
import * as posenet from './posenet/posenet'; import * as posenet from './posenet/posenet';
import * as handpose from './handpose/handpose'; import * as handpose from './handpose/handpose';
import * as blazepose from './blazepose/blazepose'; import * as blazepose from './blazepose/blazepose';
import * as movenet from './movenet/movenet';
import * as nanodet from './object/nanodet'; import * as nanodet from './object/nanodet';
import * as centernet from './object/centernet'; import * as centernet from './object/centernet';
import * as draw from './draw/draw'; import * as draw from './draw/draw';
@ -80,6 +81,7 @@ export declare class Human {
posenet: Model | null; posenet: Model | null;
blazepose: Model | null; blazepose: Model | null;
efficientpose: Model | null; efficientpose: Model | null;
movenet: Model | null;
handpose: [Model, Model] | null; handpose: [Model, Model] | null;
iris: Model | null; iris: Model | null;
age: Model | null; age: Model | null;
@ -94,7 +96,7 @@ export declare class Human {
classes: { classes: {
facemesh: typeof facemesh; facemesh: typeof facemesh;
emotion: typeof emotion; emotion: typeof emotion;
body: typeof posenet | typeof blazepose; body: typeof posenet | typeof blazepose | typeof movenet;
hand: typeof handpose; hand: typeof handpose;
nanodet: typeof nanodet; nanodet: typeof nanodet;
centernet: typeof centernet; centernet: typeof centernet;

6
types/movenet/movenet.d.ts vendored Normal file
View File

@ -0,0 +1,6 @@
/**
* EfficientPose Module
*/
import { Body } from '../result';
export declare function load(config: any): Promise<any>;
export declare function predict(image: any, config: any): Promise<Body[]>;

2
wiki

@ -1 +1 @@
Subproject commit 30aee32aa12b52d46e663488d1d0124bfb73f527 Subproject commit 317a8fc76cd933cc38f59948ffade324fc8f1df2