diff --git a/CHANGELOG.md b/CHANGELOG.md index 84327e95..135b2dee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/03/26 mandic00@live.com + + ### **1.2.5** 2021/03/25 mandic00@live.com - fix broken exports diff --git a/demo/browser.js b/demo/browser.js index 17af3b4a..934ffe7d 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -3,22 +3,20 @@ import Human from '../src/human'; import Menu from './menu.js'; import GLBench from './gl-bench.js'; -const userConfig = { backend: 'webgl' }; // add any user configuration overrides +// const userConfig = { backend: 'webgl' }; // add any user configuration overrides -/* const userConfig = { - backend: 'wasm', + backend: 'webgl', async: false, warmup: 'full', videoOptimized: true, filter: { enabled: true }, - face: { enabled: true, mesh: { enabled: true }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } }, + face: { enabled: false, mesh: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } }, hand: { enabled: false }, gesture: { enabled: false }, - body: { enabled: false, modelPath: '../models/blazepose.json' }, + body: { enabled: true, modelPath: '../models/efficientpose.json' }, object: { enabled: false }, }; -*/ const human = new Human(userConfig); diff --git a/package.json b/package.json index 3931d150..0e483721 100644 --- a/package.json +++ b/package.json @@ -56,14 +56,14 @@ "@tensorflow/tfjs-layers": "^3.3.0", "@tensorflow/tfjs-node": "^3.3.0", "@tensorflow/tfjs-node-gpu": "^3.3.0", - "@types/node": "^14.14.35", + "@types/node": "^14.14.36", "@typescript-eslint/eslint-plugin": "^4.19.0", "@typescript-eslint/parser": "^4.19.0", "@vladmandic/pilogger": "^0.2.15", "chokidar": "^3.5.1", "dayjs": "^1.10.4", - "esbuild": "^0.10.0", - "eslint": "^7.22.0", + "esbuild": "^0.10.1", + "eslint": "^7.23.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 35e292cd..4ba78c34 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -221,12 +221,14 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { } if (drawOptions.drawLabels) { ctx.font = drawOptions.font; - for (const pt of result[i].keypoints) { - ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color; - ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4); + if (result[i].keypoints) { + for (const pt of result[i].keypoints) { + ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color; + ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4); + } } } - if (drawOptions.drawPolygons) { + if (drawOptions.drawPolygons && result[i].keypoints) { let part; const points: any[] = []; // torso diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts new file mode 100644 index 00000000..1fad3b1d --- /dev/null +++ b/src/efficientpose/efficientpose.ts @@ -0,0 +1,97 @@ +import { log } from '../helpers'; +import * as tf from '../../dist/tfjs.esm.js'; +import * as profile from '../profile'; + +let model; +let last = { }; +let skipped = Number.MAX_SAFE_INTEGER; + +const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle']; + +export async function load(config) { + if (!model) { + model = await tf.loadGraphModel(config.body.modelPath); + if (config.debug) log(`load model: ${config.body.modelPath.match(/\/(.*)\./)[1]}`); + } + return model; +} + +// performs argmax and max functions on a 2d tensor +function max2d(inputs, minScore) { + const [width, height] = inputs.shape; + return tf.tidy(() => { + // modulus op implemented in tf + const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); + // combine all data + const reshaped = tf.reshape(inputs, [height * width]); + // get highest score + const score = tf.max(reshaped, 0).dataSync()[0]; + if (score > minScore) { + // skip coordinate calculation is score is too low + const coords = tf.argMax(reshaped, 0); + const x = mod(coords, width).dataSync()[0]; + const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; + return [x, y, score]; + } + return [0, 0, score]; + }); +} + +export async function predict(image, config) { + if (!model) return null; + if ((skipped < config.body.skipFrames) && config.videoOptimized && Object.keys(last).length > 0) { + skipped++; + return last; + } + if (config.videoOptimized) skipped = 0; + else skipped = Number.MAX_SAFE_INTEGER; + return new Promise(async (resolve) => { + const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); + const enhance = tf.mul(resize, [255.0]); + tf.dispose(resize); + + let resT; + + if (!config.profile) { + if (config.body.enabled) resT = await model.predict(enhance); + } else { + const profileT = config.body.enabled ? await tf.profile(() => model.predict(enhance)) : {}; + resT = profileT.result.clone(); + profileT.result.dispose(); + profile.run('body', profileT); + } + enhance.dispose(); + + if (resT) { + const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = []; + const squeeze = resT.squeeze(); + tf.dispose(resT); + // body parts are basically just a stack of 2d tensors + const stack = squeeze.unstack(2); + tf.dispose(squeeze); + // process each unstacked tensor as a separate body part + for (let id = 0; id < stack.length; id++) { + // actual processing to get coordinates and score + const [x, y, score] = max2d(stack[id], config.body.scoreThreshold); + if (score > config.body.scoreThreshold) { + parts.push({ + id, + score, + part: bodyParts[id], + positionRaw: { + xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1 + yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1 + }, + position: { + x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size + y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), // y normalized to input image size + }, + }); + } + } + stack.forEach((s) => tf.dispose(s)); + last = parts; + } + resolve(last); + }); +} diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 13644d54..5c53c899 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -55,7 +55,7 @@ export class HandPose { } } -export async function load(config) { +export async function load(config): Promise { const [handDetectorModel, handPoseModel] = await Promise.all([ config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null, config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null, diff --git a/src/human.ts b/src/human.ts index 173a0cc4..9a914b20 100644 --- a/src/human.ts +++ b/src/human.ts @@ -12,6 +12,7 @@ import * as embedding from './embedding/embedding'; import * as posenet from './posenet/posenet'; import * as handpose from './handpose/handpose'; import * as blazepose from './blazepose/blazepose'; +import * as efficientpose from './efficientpose/efficientpose'; import * as nanodet from './nanodet/nanodet'; import * as gesture from './gesture/gesture'; import * as image from './image/image'; @@ -62,9 +63,10 @@ export class Human { }; // models models: { - face: facemesh.MediaPipeFaceMesh | null, + face: facemesh.MediaPipeFaceMesh | Model | null, posenet: posenet.PoseNet | null, blazepose: Model | null, + efficientpose: Model | null, handpose: handpose.HandPose | null, iris: Model | null, age: Model | null, @@ -108,6 +110,7 @@ export class Human { face: null, posenet: null, blazepose: null, + efficientpose: null, handpose: null, iris: null, age: null, @@ -206,9 +209,12 @@ export class Human { this.models.gender, this.models.emotion, this.models.embedding, + // @ts-ignore this.models.handpose, + // @ts-ignore false warning with latest @typescript-eslint this.models.posenet, this.models.blazepose, + this.models.efficientpose, this.models.nanodet, this.models.faceres, ] = await Promise.all([ @@ -217,9 +223,10 @@ export class Human { this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null), this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null), this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null), - this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null), + this.models.handpose || (this.config.hand.enabled ? >handpose.load(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null), - this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null), + this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null), + this.models.efficientpose || (this.config.body.enabled && this.config.body.modelPath.includes('efficientpose') ? efficientpose.load(this.config) : null), this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null), this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null), ]); @@ -232,6 +239,7 @@ export class Human { if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config); if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config); if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config); + if (this.config.body.enabled && !this.models.efficientpose && this.config.body.modelPath.includes('efficientpose')) this.models.efficientpose = await efficientpose.load(this.config); if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config); if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config); } @@ -363,13 +371,15 @@ export class Human { this.analyze('Start Body:'); if (this.config.async) { if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : []; - else bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : []; if (this.perf.body) delete this.perf.body; } else { this.state = 'run:body'; timeStamp = now(); if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : []; - else bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : []; + else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : []; current = Math.trunc(now() - timeStamp); if (current > 0) this.perf.body = current; }