rewrite posenet decoder

pull/134/head
Vladimir Mandic 2021-04-24 16:04:49 -04:00
parent 951fae2322
commit 65cabb2693
31 changed files with 962 additions and 1117 deletions

View File

@ -12,8 +12,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/04/24 mandic00@live.com
### **origin/main** 2021/04/22 mandic00@live.com
### **origin/main** 2021/04/24 mandic00@live.com
- remove efficientpose
- major version rebuild
### **1.6.1** 2021/04/22 mandic00@live.com

View File

@ -18,17 +18,17 @@ const userConfig = {
warmup: 'full',
videoOptimized: false,
filter: {
enabled: true,
enabled: false,
flip: false,
},
face: { enabled: true,
face: { enabled: false,
mesh: { enabled: true },
iris: { enabled: true },
description: { enabled: false },
emotion: { enabled: false },
},
hand: { enabled: false },
gesture: { enabled: true },
gesture: { enabled: false },
body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' },
// object: { enabled: true },

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

692
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

692
dist/human.js vendored

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

20
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -151,3 +151,37 @@
2021-04-24 11:48:51 INFO:  Generate types: ["src/human.ts"]
2021-04-24 11:48:56 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-24 11:48:56 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-24 16:00:49 INFO:  @vladmandic/human version 1.6.1
2021-04-24 16:00:49 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-04-24 16:00:49 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-24 16:00:49 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:00:49 STATE: Build for: node type: node: {"imports":40,"importBytes":542104,"outputBytes":304213,"outputFiles":"dist/human.node.js"}
2021-04-24 16:00:49 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:00:49 STATE: Build for: nodeGPU type: node: {"imports":40,"importBytes":542108,"outputBytes":304221,"outputFiles":"dist/human.node-gpu.js"}
2021-04-24 16:00:49 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:00:49 STATE: Build for: nodeWASM type: node: {"imports":40,"importBytes":542154,"outputBytes":304265,"outputFiles":"dist/human.node-wasm.js"}
2021-04-24 16:00:49 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:00:49 STATE: Build for: browserNoBundle type: esm: {"imports":40,"importBytes":542765,"outputBytes":304246,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-24 16:00:50 STATE: Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:00:51 STATE: Build for: browserBundle type: iife: {"imports":40,"importBytes":1643099,"outputBytes":1402155,"outputFiles":"dist/human.js"}
2021-04-24 16:00:51 STATE: Build for: browserBundle type: esm: {"imports":40,"importBytes":1643099,"outputBytes":1402113,"outputFiles":"dist/human.esm.js"}
2021-04-24 16:00:51 INFO:  Generate types: ["src/human.ts"]
2021-04-24 16:00:57 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-24 16:00:57 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-04-24 16:03:56 INFO:  @vladmandic/human version 1.6.1
2021-04-24 16:03:56 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-04-24 16:03:56 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
2021-04-24 16:03:56 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:03:56 STATE: Build for: node type: node: {"imports":40,"importBytes":542082,"outputBytes":304213,"outputFiles":"dist/human.node.js"}
2021-04-24 16:03:56 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:03:57 STATE: Build for: nodeGPU type: node: {"imports":40,"importBytes":542086,"outputBytes":304221,"outputFiles":"dist/human.node-gpu.js"}
2021-04-24 16:03:57 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:03:57 STATE: Build for: nodeWASM type: node: {"imports":40,"importBytes":542132,"outputBytes":304265,"outputFiles":"dist/human.node-wasm.js"}
2021-04-24 16:03:57 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:03:57 STATE: Build for: browserNoBundle type: esm: {"imports":40,"importBytes":542743,"outputBytes":304246,"outputFiles":"dist/human.esm-nobundle.js"}
2021-04-24 16:03:57 STATE: Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
2021-04-24 16:03:58 STATE: Build for: browserBundle type: iife: {"imports":40,"importBytes":1643077,"outputBytes":1402155,"outputFiles":"dist/human.js"}
2021-04-24 16:03:58 STATE: Build for: browserBundle type: esm: {"imports":40,"importBytes":1643077,"outputBytes":1402113,"outputFiles":"dist/human.esm.js"}
2021-04-24 16:03:58 INFO:  Generate types: ["src/human.ts"]
2021-04-24 16:04:03 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-04-24 16:04:03 INFO:  Generate TypeDocs: ["src/human.ts"]

View File

@ -366,7 +366,7 @@ const config: Config = {
maxDetections: 1, // maximum number of people detected in the input
// should be set to the minimum number for performance
// only valid for posenet as blazepose only detects single pose
scoreThreshold: 0.3, // threshold for deciding when to remove boxes based on score
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
// in non-maximum suppression
// only valid for posenet as blazepose only detects single pose
nmsRadius: 20, // radius for deciding points are too close in non-maximum suppression

View File

@ -88,7 +88,7 @@ export class Human {
/** Internal: Currently loaded models */
models: {
face: facemesh.MediaPipeFaceMesh | Model | null,
posenet: posenet.PoseNet | null,
posenet: Model | null,
blazepose: Model | null,
efficientpose: Model | null,
handpose: handpose.HandPose | null,
@ -444,14 +444,14 @@ export class Human {
// run body: can be posenet or blazepose
this.analyze('Start Body:');
if (this.config.async) {
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
if (this.perf.body) delete this.perf.body;
} else {
this.state = 'run:body';
timeStamp = now();
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp);

View File

@ -1,78 +0,0 @@
import * as decodeSingle from './decodeSingle';
import * as utils from './utils';
const kLocalMaximumRadius = 1;
const defaultOutputStride = 16;
function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores) {
const [height, width] = scores.shape;
let localMaximum = true;
const yStart = Math.max(heatmapY - localMaximumRadius, 0);
const yEnd = Math.min(heatmapY + localMaximumRadius + 1, height);
for (let yCurrent = yStart; yCurrent < yEnd; ++yCurrent) {
const xStart = Math.max(heatmapX - localMaximumRadius, 0);
const xEnd = Math.min(heatmapX + localMaximumRadius + 1, width);
for (let xCurrent = xStart; xCurrent < xEnd; ++xCurrent) {
if (scores.get(yCurrent, xCurrent, keypointId) > score) {
localMaximum = false;
break;
}
}
if (!localMaximum) break;
}
return localMaximum;
}
export function buildPartWithScoreQueue(scoreThreshold, localMaximumRadius, scores) {
const [height, width, numKeypoints] = scores.shape;
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
const score = scores.get(heatmapY, heatmapX, keypointId);
// Only consider parts with score greater or equal to threshold as root candidates.
if (score < scoreThreshold) continue;
// Only consider keypoints whose score is maximum in a local window.
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores)) {
queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
}
}
}
}
return queue;
}
function withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, { x, y }, keypointId) {
return poses.some(({ keypoints }) => {
const correspondingKeypoint = keypoints[keypointId].position;
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
});
}
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
if (!withinNmsRadiusOfCorrespondingPoint(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
return result;
}, 0.0);
return notOverlappedKeypointScores / instanceKeypoints.length;
}
export function decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
const queue = buildPartWithScoreQueue(scoreThreshold, kLocalMaximumRadius, scoresBuffer);
const squaredNmsRadius = nmsRadius ^ 2;
// Generate at most maxDetections object instances per image in decreasing root part score order.
while (poses.length < maxDetections && !queue.empty()) {
// The top element in the queue is the next root candidate.
const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
if (withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root.
const keypoints = decodeSingle.decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
const box = utils.getBoundingBox(keypoints);
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
}
return poses;
}

View File

@ -1,48 +0,0 @@
import * as tf from '../../dist/tfjs.esm.js';
import * as keypoints from './keypoints';
export function getPointsConfidence(heatmapScores, heatMapCoords) {
const numKeypoints = keypoints.count; // also in heatMapCoords.shape[0];
const result:Array<number> = [];
for (let keypoint = 0; keypoint < numKeypoints; keypoint++) {
const y = heatMapCoords.get(keypoint, 0);
const x = heatMapCoords.get(keypoint, 1);
result.push(heatmapScores.get(y, x, keypoint));
}
return result;
}
export function getOffsetPoints(heatMapCoordsBuffer, outputStride, offsetsBuffer) {
const getOffsetPoint = (y, x, keypoint) => ([
offsetsBuffer.get(y, x, keypoint),
offsetsBuffer.get(y, x, keypoint + keypoints.count),
]);
const getOffsetVectors = () => {
const result: Array<number[]> = [];
for (let keypoint = 0; keypoint < keypoints.count; keypoint++) {
const heatmapY = heatMapCoordsBuffer.get(keypoint, 0);
const heatmapX = heatMapCoordsBuffer.get(keypoint, 1);
result.push(getOffsetPoint(heatmapY, heatmapX, keypoint));
}
return result;
};
return tf.tidy(() => heatMapCoordsBuffer.toTensor().mul(tf.scalar(outputStride, 'int32')).toFloat().add(getOffsetVectors()));
}
export function argmax2d(inputs) {
const mod = (a, b) => tf.tidy(() => {
const floored = a.div(tf.scalar(b, 'int32'));
return a.sub(floored.mul(tf.scalar(b, 'int32')));
});
const [height, width, depth] = inputs.shape;
return tf.tidy(() => {
const reshaped = inputs.reshape([height * width, depth]);
const coords = reshaped.argMax(0);
const yCoords = coords.div(tf.scalar(width, 'int32')).expandDims(1);
const xCoords = mod(coords, width).expandDims(1);
return tf.concat([yCoords, xCoords], 1);
});
}

View File

@ -1,104 +0,0 @@
import * as keypoints from './keypoints';
import * as decoders from './decodeParts';
import * as utils from './utils';
const parentChildrenTuples = keypoints.poseChain.map(([parentJoinName, childJoinName]) => ([keypoints.partIds[parentJoinName], keypoints.partIds[childJoinName]]));
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const defaultOutputStride = 16;
function getDisplacement(edgeId, point, displacements) {
const numEdges = displacements.shape[2] / 2;
return {
y: displacements.get(point.y, point.x, edgeId),
x: displacements.get(point.y, point.x, numEdges + edgeId),
};
}
function getStridedIndexNearPoint(point, outputStride, height, width) {
return {
y: utils.clamp(Math.round(point.y / outputStride), 0, height - 1),
x: utils.clamp(Math.round(point.x / outputStride), 0, width - 1),
};
}
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
const [height, width] = scoresBuffer.shape;
// Nearest neighbor interpolation for the source->target displacements.
const sourceKeypointIndices = getStridedIndexNearPoint(sourceKeypoint.position, outputStride, height, width);
const displacement = getDisplacement(edgeId, sourceKeypointIndices, displacements);
const displacedPoint = utils.addVectors(sourceKeypoint.position, displacement);
let targetKeypoint = displacedPoint;
for (let i = 0; i < offsetRefineStep; i++) {
const targetKeypointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width);
const offsetPoint = utils.getOffsetPoint(targetKeypointIndices.y, targetKeypointIndices.x, targetKeypointId, offsets);
targetKeypoint = utils.addVectors({
x: targetKeypointIndices.x * outputStride,
y: targetKeypointIndices.y * outputStride,
}, { x: offsetPoint.x, y: offsetPoint.y });
}
const targetKeyPointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width);
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
return { position: targetKeypoint, part: keypoints.partNames[targetKeypointId], score };
}
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
const numParts = scores.shape[2];
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
// Start a new detection instance at the position of the root.
const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: keypoints.partNames[rootPart.id],
position: rootPoint,
};
// Decode the part positions upwards in the tree, following the backward displacements.
for (let edge = numEdges - 1; edge >= 0; --edge) {
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
}
}
// Decode the part positions downwards in the tree, following the forward displacements.
for (let edge = 0; edge < numEdges; ++edge) {
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
}
}
return instanceKeypoints;
}
export async function decodeSinglePose(heatmapScores, offsets, minScore) {
const heatmapValues = decoders.argmax2d(heatmapScores);
const allTensorBuffers = await Promise.all([heatmapScores.buffer(), offsets.buffer(), heatmapValues.buffer()]);
const scoresBuffer = allTensorBuffers[0];
const offsetsBuffer = allTensorBuffers[1];
const heatmapValuesBuffer = allTensorBuffers[2];
const offsetPoints = decoders.getOffsetPoints(heatmapValuesBuffer, defaultOutputStride, offsetsBuffer);
const offsetPointsData = offsetPoints.dataSync();
const keypointConfidence = decoders.getPointsConfidence(scoresBuffer, heatmapValuesBuffer);
let avgScore = 0.0;
const filteredKeypoints = keypointConfidence
.filter((score) => score > minScore)
.map((score, i) => {
avgScore += score;
return {
position: {
y: offsetPointsData[2 * i + 0], // offsetPointsBuffer.get(i, 0),
x: offsetPointsData[2 * i + 1], // offsetPointsBuffer.get(i, 1),
},
part: keypoints.partNames[i],
score,
};
});
heatmapValues.dispose();
offsetPoints.dispose();
const box = utils.getBoundingBox(filteredKeypoints);
return { keypoints: filteredKeypoints, box, score: Math.round(100 * avgScore / filteredKeypoints.length) / 100 };
}

View File

@ -1,61 +1,28 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as posenetModel from './posenetModel';
import * as decodeMultiple from './decodeMultiple';
import * as decodeSingle from './decodeSingle';
import * as poses from './poses';
import * as util from './utils';
let model;
const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd'/* offsets */, 'MobilenetV1/heatmap_2/BiasAdd'/* heatmapScores */, 'MobilenetV1/displacement_fwd_2/BiasAdd'/* displacementFwd */, 'MobilenetV1/displacement_bwd_2/BiasAdd'/* displacementBwd */];
async function estimateMultiple(input, res, config, inputSize) {
const toTensorBuffers3D = (tensors) => Promise.all(tensors.map((tensor) => tensor.buffer()));
return new Promise(async (resolve) => {
const allTensorBuffers = await toTensorBuffers3D([res.heatmapScores, res.offsets, res.displacementFwd, res.displacementBwd]);
const scoresBuffer = allTensorBuffers[0];
const offsetsBuffer = allTensorBuffers[1];
const displacementsFwdBuffer = allTensorBuffers[2];
const displacementsBwdBuffer = allTensorBuffers[3];
const poses = await decodeMultiple.decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, config.body.nmsRadius, config.body.maxDetections, config.body.scoreThreshold);
const scaled = util.scalePoses(poses, [input.shape[1], input.shape[2]], [inputSize, inputSize]);
resolve(scaled);
export async function predict(input, config) {
const res = tf.tidy(() => {
const resized = input.resizeBilinear([model.inputs[0].shape[2], model.inputs[0].shape[1]]);
const normalized = resized.toFloat().div(127.5).sub(1.0);
const results = model.execute(normalized, poseNetOutputs);
const results3d = results.map((y) => y.squeeze([0]));
results3d[1] = results3d[1].sigmoid(); // apply sigmoid on scores
return results3d;
});
}
async function estimateSingle(input, res, config, inputSize) {
return new Promise(async (resolve) => {
const pose = await decodeSingle.decodeSinglePose(res.heatmapScores, res.offsets, config.body.scoreThreshold);
const scaled = util.scalePoses([pose], [input.shape[1], input.shape[2]], [inputSize, inputSize]);
resolve(scaled);
});
}
const buffers = await Promise.all(res.map((tensor) => tensor.buffer()));
for (const t of res) t.dispose();
export class PoseNet {
baseModel: any;
inputSize: number
constructor(baseModel) {
this.baseModel = baseModel;
this.inputSize = baseModel.model.inputs[0].shape[1];
}
const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.nmsRadius, config.body.maxDetections, config.body.scoreThreshold);
const scaled = util.scalePoses(decoded, [input.shape[1], input.shape[2]], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
async estimatePoses(input, config) {
const res = this.baseModel.predict(input, config);
const poses = (config.body.maxDetections < 2)
? await estimateSingle(input, res, config, this.inputSize)
: await estimateMultiple(input, res, config, this.inputSize);
res.heatmapScores.dispose();
res.offsets.dispose();
res.displacementFwd.dispose();
res.displacementBwd.dispose();
return poses;
}
dispose() {
this.baseModel.dispose();
}
return scaled;
}
export async function load(config) {
@ -64,7 +31,5 @@ export async function load(config) {
if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
} else if (config.debug) log('cached model:', model.modelUrl);
const mobilenet = new posenetModel.BaseModel(model);
const poseNet = new PoseNet(mobilenet);
return poseNet;
return model;
}

View File

@ -1,38 +0,0 @@
import * as tf from '../../dist/tfjs.esm.js';
const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd', 'MobilenetV1/heatmap_2/BiasAdd', 'MobilenetV1/displacement_fwd_2/BiasAdd', 'MobilenetV1/displacement_bwd_2/BiasAdd'];
function nameOutputResultsMobileNet(results) {
const [offsets, heatmap, displacementFwd, displacementBwd] = results;
return { offsets, heatmap, displacementFwd, displacementBwd };
}
export class BaseModel {
model: any;
inputSize: number;
constructor(model) {
this.model = model;
this.inputSize = model.inputs[0].shape[1];
}
predict(input) {
return tf.tidy(() => {
const resized = input.resizeBilinear([this.inputSize, this.inputSize]);
const normalized = resized.toFloat().div(127.5).sub(1.0);
// const asBatch = asFloat.expandDims(0);
const results = this.model.execute(normalized, poseNetOutputs);
const results3d = results.map((y) => y.squeeze([0]));
const namedResults = nameOutputResultsMobileNet(results3d);
return {
heatmapScores: namedResults.heatmap.sigmoid(),
offsets: namedResults.offsets,
displacementFwd: namedResults.displacementFwd,
displacementBwd: namedResults.displacementBwd,
};
});
}
dispose() {
this.model.dispose();
}
}

140
src/posenet/poses.ts Normal file
View File

@ -0,0 +1,140 @@
import * as utils from './utils';
import * as kpt from './keypoints';
const localMaximumRadius = 1;
const defaultOutputStride = 16;
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
const getDisplacement = (point) => ({
y: displacements.get(point.y, point.x, edgeId),
x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId),
});
const getStridedIndexNearPoint = (point, height, width) => ({
y: utils.clamp(Math.round(point.y / outputStride), 0, height - 1),
x: utils.clamp(Math.round(point.x / outputStride), 0, width - 1),
});
const [height, width] = scoresBuffer.shape;
// Nearest neighbor interpolation for the source->target displacements.
const sourceKeypointIndices = getStridedIndexNearPoint(sourceKeypoint.position, height, width);
const displacement = getDisplacement(sourceKeypointIndices);
const displacedPoint = utils.addVectors(sourceKeypoint.position, displacement);
let targetKeypoint = displacedPoint;
for (let i = 0; i < offsetRefineStep; i++) {
const targetKeypointIndices = getStridedIndexNearPoint(targetKeypoint, height, width);
const offsetPoint = utils.getOffsetPoint(targetKeypointIndices.y, targetKeypointIndices.x, targetKeypointId, offsets);
targetKeypoint = utils.addVectors({
x: targetKeypointIndices.x * outputStride,
y: targetKeypointIndices.y * outputStride,
}, { x: offsetPoint.x, y: offsetPoint.y });
}
const targetKeyPointIndices = getStridedIndexNearPoint(targetKeypoint, height, width);
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score };
}
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]]));
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
const numParts = scores.shape[2];
const numEdges = parentToChildEdges.length;
const instanceKeypoints = new Array(numParts);
// Start a new detection instance at the position of the root.
const { part: rootPart, score: rootScore } = root;
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
instanceKeypoints[rootPart.id] = {
score: rootScore,
part: kpt.partNames[rootPart.id],
position: rootPoint,
};
// Decode the part positions upwards in the tree, following the backward displacements.
for (let edge = numEdges - 1; edge >= 0; --edge) {
const sourceKeypointId = parentToChildEdges[edge];
const targetKeypointId = childToParentEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
}
}
// Decode the part positions downwards in the tree, following the forward displacements.
for (let edge = 0; edge < numEdges; ++edge) {
const sourceKeypointId = childToParentEdges[edge];
const targetKeypointId = parentToChildEdges[edge];
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
}
}
return instanceKeypoints;
}
function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores) {
const [height, width] = scores.shape;
let localMaximum = true;
const yStart = Math.max(heatmapY - localMaximumRadius, 0);
const yEnd = Math.min(heatmapY + localMaximumRadius + 1, height);
for (let yCurrent = yStart; yCurrent < yEnd; ++yCurrent) {
const xStart = Math.max(heatmapX - localMaximumRadius, 0);
const xEnd = Math.min(heatmapX + localMaximumRadius + 1, width);
for (let xCurrent = xStart; xCurrent < xEnd; ++xCurrent) {
if (scores.get(yCurrent, xCurrent, keypointId) > score) {
localMaximum = false;
break;
}
}
if (!localMaximum) break;
}
return localMaximum;
}
export function buildPartWithScoreQueue(scoreThreshold, scores) {
const [height, width, numKeypoints] = scores.shape;
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
const score = scores.get(heatmapY, heatmapX, keypointId);
// Only consider parts with score greater or equal to threshold as root candidates.
if (score < scoreThreshold) continue;
// Only consider keypoints whose score is maximum in a local window.
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
}
}
}
return queue;
}
function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) {
return poses.some(({ keypoints }) => {
const correspondingKeypoint = keypoints[keypointId].position;
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
});
}
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
return result;
}, 0.0);
return notOverlappedKeypointScores / instanceKeypoints.length;
}
export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer);
const squaredNmsRadius = nmsRadius ** 2;
// Generate at most maxDetections object instances per image in decreasing root part score order.
while (poses.length < maxDetections && !queue.empty()) {
// The top element in the queue is the next root candidate.
const root = queue.dequeue();
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
// Else start a new detection instance at the position of the root.
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold);
const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
const box = utils.getBoundingBox(keypoints);
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
}
return poses;
}

View File

@ -452,7 +452,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
<a name="models" class="tsd-anchor"></a>
<h3>models</h3>
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-comment tsd-typography">
@ -497,7 +497,7 @@
<h5>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
<li class="tsd-parameter">
<h5>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span></h5>
<h5>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
</li>
</ul>
</div>

2
types/human.d.ts vendored
View File

@ -76,7 +76,7 @@ export declare class Human {
/** Internal: Currently loaded models */
models: {
face: facemesh.MediaPipeFaceMesh | Model | null;
posenet: posenet.PoseNet | null;
posenet: Model | null;
blazepose: Model | null;
efficientpose: Model | null;
handpose: handpose.HandPose | null;

View File

@ -1,7 +0,0 @@
import * as utils from './utils';
export declare function buildPartWithScoreQueue(scoreThreshold: any, localMaximumRadius: any, scores: any): utils.MaxHeap;
export declare function decodeMultiplePoses(scoresBuffer: any, offsetsBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, nmsRadius: any, maxDetections: any, scoreThreshold: any): {
keypoints: any;
box: any;
score: number;
}[];

View File

@ -1,3 +0,0 @@
export declare function getPointsConfidence(heatmapScores: any, heatMapCoords: any): number[];
export declare function getOffsetPoints(heatMapCoordsBuffer: any, outputStride: any, offsetsBuffer: any): any;
export declare function argmax2d(inputs: any): any;

View File

@ -1,13 +0,0 @@
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
export declare function decodeSinglePose(heatmapScores: any, offsets: any, minScore: any): Promise<{
keypoints: {
position: {
y: any;
x: any;
};
part: string;
score: number;
}[];
box: any[];
score: number;
}>;

View File

@ -1,8 +1,2 @@
export declare class PoseNet {
baseModel: any;
inputSize: number;
constructor(baseModel: any);
estimatePoses(input: any, config: any): Promise<unknown>;
dispose(): void;
}
export declare function load(config: any): Promise<PoseNet>;
export declare function predict(input: any, config: any): Promise<any>;
export declare function load(config: any): Promise<any>;

View File

@ -1,7 +0,0 @@
export declare class BaseModel {
model: any;
inputSize: number;
constructor(model: any);
predict(input: any): any;
dispose(): void;
}

8
types/posenet/poses.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
import * as utils from './utils';
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
export declare function buildPartWithScoreQueue(scoreThreshold: any, scores: any): utils.MaxHeap;
export declare function decode(offsetsBuffer: any, scoresBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, nmsRadius: any, maxDetections: any, scoreThreshold: any): {
keypoints: any;
box: any;
score: number;
}[];