From aa10a3635013124dfeb5a68c1aafad9b3de075e5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 11 Jan 2021 14:35:57 -0500 Subject: [PATCH] work on blazepose --- config.js | 10 +++++ demo/browser.js | 6 ++- src/{face => blazeface}/coords.js | 0 src/{hand => handpose}/anchors.js | 0 src/{hand => handpose}/box.js | 0 src/{hand => handpose}/handdetector.js | 0 src/{hand => handpose}/handpipeline.js | 0 src/{hand => handpose}/handpose.js | 0 src/human.js | 62 ++++++++++---------------- 9 files changed, 38 insertions(+), 40 deletions(-) rename src/{face => blazeface}/coords.js (100%) rename src/{hand => handpose}/anchors.js (100%) rename src/{hand => handpose}/box.js (100%) rename src/{hand => handpose}/handdetector.js (100%) rename src/{hand => handpose}/handpipeline.js (100%) rename src/{hand => handpose}/handpose.js (100%) diff --git a/config.js b/config.js index 218bff14..6b653ab9 100644 --- a/config.js +++ b/config.js @@ -144,6 +144,16 @@ export default { modelType: 'MobileNet', // Human includes MobileNet version, but you can switch to ResNet }, + pose: { + enabled: false, + scoreThreshold: 0.6, // threshold for deciding when to remove boxes based on score + // in non-maximum suppression + iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much + // in non-maximum suppression + modelPath: '../models/blazepose.json', + inputSize: 128, // fixed value + }, + hand: { enabled: true, rotation: false, // use best-guess rotated hand image or just box with rotation as-is diff --git a/demo/browser.js b/demo/browser.js index 437bbdbe..a7e5f3e7 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -4,11 +4,13 @@ import Menu from './menu.js'; import GLBench from './gl-bench.js'; const userConfig = {}; // add any user configuration overrides + /* const userConfig = { - face: { enabled: true, iris: { enabled: false } }, - body: { enabled: true }, + face: { enabled: false, iris: { enabled: false } }, + body: { enabled: false }, hand: { enabled: false }, + pose: { enabled: true }, }; */ diff --git a/src/face/coords.js b/src/blazeface/coords.js similarity index 100% rename from src/face/coords.js rename to src/blazeface/coords.js diff --git a/src/hand/anchors.js b/src/handpose/anchors.js similarity index 100% rename from src/hand/anchors.js rename to src/handpose/anchors.js diff --git a/src/hand/box.js b/src/handpose/box.js similarity index 100% rename from src/hand/box.js rename to src/handpose/box.js diff --git a/src/hand/handdetector.js b/src/handpose/handdetector.js similarity index 100% rename from src/hand/handdetector.js rename to src/handpose/handdetector.js diff --git a/src/hand/handpipeline.js b/src/handpose/handpipeline.js similarity index 100% rename from src/hand/handpipeline.js rename to src/handpose/handpipeline.js diff --git a/src/hand/handpose.js b/src/handpose/handpose.js similarity index 100% rename from src/hand/handpose.js rename to src/handpose/handpose.js diff --git a/src/human.js b/src/human.js index 0f20c5cb..a905a33a 100644 --- a/src/human.js +++ b/src/human.js @@ -1,13 +1,14 @@ import { log } from './log.js'; import * as tf from '../dist/tfjs.esm.js'; import * as backend from './tfjs/backend.js'; -import * as facemesh from './face/facemesh.js'; +import * as facemesh from './blazeface/facemesh.js'; import * as age from './age/age.js'; import * as gender from './gender/gender.js'; import * as emotion from './emotion/emotion.js'; import * as embedding from './embedding/embedding.js'; -import * as posenet from './body/posenet.js'; -import * as handpose from './hand/handpose.js'; +import * as posenet from './posenet/posenet.js'; +import * as blazepose from './blazepose/blazepose.js'; +import * as handpose from './handpose/handpose.js'; import * as gesture from './gesture/gesture.js'; import * as image from './image.js'; import * as profile from './profile.js'; @@ -18,7 +19,6 @@ import * as app from '../package.json'; // helper function: gets elapsed time on both browser and nodejs const now = () => { if (typeof performance !== 'undefined') return performance.now(); - // @ts-ignore return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000); }; @@ -73,7 +73,6 @@ class Human { } profile() { - // @ts-ignore if (this.config.profile) return profile.data; return {}; } @@ -104,7 +103,6 @@ class Human { } simmilarity(embedding1, embedding2) { - // @ts-ignore if (this.config.face.embedding.enabled) return embedding.simmilarity(embedding1, embedding2); return 0; } @@ -134,37 +132,26 @@ class Human { this.models.embedding, this.models.posenet, this.models.handpose, + this.models.blazepose, ] = await Promise.all([ - // @ts-ignore this.models.facemesh || (this.config.face.enabled ? facemesh.load(this.config) : null), - // @ts-ignore this.models.age || ((this.config.face.enabled && this.config.face.age.enabled) ? age.load(this.config) : null), - // @ts-ignore this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null), - // @ts-ignore this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null), - // @ts-ignore this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null), - // @ts-ignore this.models.posenet || (this.config.body.enabled ? posenet.load(this.config) : null), - // @ts-ignore this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null), + this.models.blazepose || (this.config.pose.enabled ? blazepose.load(this.config) : null), ]); } else { - // @ts-ignore if (this.config.face.enabled && !this.models.facemesh) this.models.facemesh = await facemesh.load(this.config); - // @ts-ignore if (this.config.face.enabled && this.config.face.age.enabled && !this.models.age) this.models.age = await age.load(this.config); - // @ts-ignore if (this.config.face.enabled && this.config.face.gender.enabled && !this.models.gender) this.models.gender = await gender.load(this.config); - // @ts-ignore if (this.config.face.enabled && this.config.face.emotion.enabled && !this.models.emotion) this.models.emotion = await emotion.load(this.config); - // @ts-ignore if (this.config.face.enabled && this.config.face.embedding.enabled && !this.models.embedding) this.models.embedding = await embedding.load(this.config); - // @ts-ignore if (this.config.body.enabled && !this.models.posenet) this.models.posenet = await posenet.load(this.config); - // @ts-ignore if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config); + if (this.config.pose.enabled && !this.models.blazepose) this.models.blazepose = await blazepose.load(this.config); } const current = Math.trunc(now() - timeStamp); if (current > (this.perf.load || 0)) this.perf.load = current; @@ -230,7 +217,6 @@ class Human { const faceRes = []; this.state = 'run:face'; timeStamp = now(); - // @ts-ignore const faces = await this.models.facemesh?.estimateFaces(input, this.config); this.perf.face = Math.trunc(now() - timeStamp); for (const face of faces) { @@ -245,12 +231,10 @@ class Human { // run age, inherits face from blazeface this.analyze('Start Age:'); if (this.config.async) { - // @ts-ignore ageRes = this.config.face.age.enabled ? age.predict(face.image, this.config) : {}; } else { this.state = 'run:age'; timeStamp = now(); - // @ts-ignore ageRes = this.config.face.age.enabled ? await age.predict(face.image, this.config) : {}; this.perf.age = Math.trunc(now() - timeStamp); } @@ -258,12 +242,10 @@ class Human { // run gender, inherits face from blazeface this.analyze('Start Gender:'); if (this.config.async) { - // @ts-ignore genderRes = this.config.face.gender.enabled ? gender.predict(face.image, this.config) : {}; } else { this.state = 'run:gender'; timeStamp = now(); - // @ts-ignore genderRes = this.config.face.gender.enabled ? await gender.predict(face.image, this.config) : {}; this.perf.gender = Math.trunc(now() - timeStamp); } @@ -271,12 +253,10 @@ class Human { // run emotion, inherits face from blazeface this.analyze('Start Emotion:'); if (this.config.async) { - // @ts-ignore emotionRes = this.config.face.emotion.enabled ? emotion.predict(face.image, this.config) : {}; } else { this.state = 'run:emotion'; timeStamp = now(); - // @ts-ignore emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {}; this.perf.emotion = Math.trunc(now() - timeStamp); } @@ -285,12 +265,10 @@ class Human { // run emotion, inherits face from blazeface this.analyze('Start Embedding:'); if (this.config.async) { - // @ts-ignore embeddingRes = this.config.face.embedding.enabled ? embedding.predict(face.image, this.config) : {}; } else { this.state = 'run:embedding'; timeStamp = now(); - // @ts-ignore embeddingRes = this.config.face.embedding.enabled ? await embedding.predict(face.image, this.config) : {}; this.perf.embedding = Math.trunc(now() - timeStamp); } @@ -346,7 +324,6 @@ class Human { async image(input, userConfig = {}) { this.state = 'image'; this.config = mergeDeep(this.config, userConfig); - // @ts-ignore const process = image.process(input, this.config); process.tensor.dispose(); return process.canvas; @@ -371,6 +348,7 @@ class Human { } let poseRes; + let blazeposeRes; let handRes; let faceRes; @@ -386,7 +364,6 @@ class Human { this.analyze('Start Scope:'); timeStamp = now(); - // @ts-ignore const process = image.process(input, this.config); if (!process || !process.tensor) { log('could not convert input to tensor'); @@ -410,36 +387,45 @@ class Human { // run posenet this.analyze('Start Body:'); if (this.config.async) { - // @ts-ignore poseRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : []; if (this.perf.body) delete this.perf.body; } else { this.state = 'run:body'; timeStamp = now(); - // @ts-ignore poseRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : []; this.perf.body = Math.trunc(now() - timeStamp); } this.analyze('End Body:'); + // run posenet + this.analyze('Start Pose:'); + if (this.config.async) { + blazeposeRes = this.config.pose.enabled ? blazepose.predict(process.tensor, this.config) : []; + if (this.perf.pose) delete this.perf.pose; + } else { + this.state = 'run:pose'; + timeStamp = now(); + blazeposeRes = this.config.pose.enabled ? await blazepose.predict(process.tensor, this.config) : []; + this.perf.pose = Math.trunc(now() - timeStamp); + } + this.analyze('End Pose:'); + // run handpose this.analyze('Start Hand:'); if (this.config.async) { - // @ts-ignore handRes = this.config.hand.enabled ? this.models.handpose?.estimateHands(process.tensor, this.config) : []; if (this.perf.hand) delete this.perf.hand; } else { this.state = 'run:hand'; timeStamp = now(); - // @ts-ignore handRes = this.config.hand.enabled ? await this.models.handpose?.estimateHands(process.tensor, this.config) : []; this.perf.hand = Math.trunc(now() - timeStamp); } - // this.analyze('End Hand:'); + this.analyze('End Hand:'); // if async wait for results if (this.config.async) { - [faceRes, poseRes, handRes] = await Promise.all([faceRes, poseRes, handRes]); + [faceRes, poseRes, blazeposeRes, handRes] = await Promise.all([faceRes, poseRes, blazeposeRes, handRes]); } process.tensor.dispose(); @@ -456,7 +442,7 @@ class Human { this.perf.total = Math.trunc(now() - timeStart); this.state = 'idle'; - resolve({ face: faceRes, body: poseRes, hand: handRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas }); + resolve({ face: faceRes, body: poseRes, hand: handRes, pose: blazeposeRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas }); }); }