mirror of https://github.com/vladmandic/human
rewrite posenet decoder
parent
951fae2322
commit
65cabb2693
|
@ -12,8 +12,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
### **HEAD -> main** 2021/04/24 mandic00@live.com
|
||||
|
||||
|
||||
### **origin/main** 2021/04/22 mandic00@live.com
|
||||
### **origin/main** 2021/04/24 mandic00@live.com
|
||||
|
||||
- remove efficientpose
|
||||
- major version rebuild
|
||||
|
||||
### **1.6.1** 2021/04/22 mandic00@live.com
|
||||
|
||||
|
|
|
@ -18,17 +18,17 @@ const userConfig = {
|
|||
warmup: 'full',
|
||||
videoOptimized: false,
|
||||
filter: {
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: true,
|
||||
face: { enabled: false,
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
description: { enabled: false },
|
||||
emotion: { enabled: false },
|
||||
},
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: true, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
// object: { enabled: true },
|
||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -151,3 +151,37 @@
|
|||
2021-04-24 11:48:51 [36mINFO: [39m Generate types: ["src/human.ts"]
|
||||
2021-04-24 11:48:56 [36mINFO: [39m Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-04-24 11:48:56 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"]
|
||||
2021-04-24 16:00:49 [36mINFO: [39m @vladmandic/human version 1.6.1
|
||||
2021-04-24 16:00:49 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.0.0
|
||||
2021-04-24 16:00:49 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: node type: node: {"imports":40,"importBytes":542104,"outputBytes":304213,"outputFiles":"dist/human.node.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: nodeGPU type: node: {"imports":40,"importBytes":542108,"outputBytes":304221,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: nodeWASM type: node: {"imports":40,"importBytes":542154,"outputBytes":304265,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:00:49 [35mSTATE:[39m Build for: browserNoBundle type: esm: {"imports":40,"importBytes":542765,"outputBytes":304246,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-04-24 16:00:50 [35mSTATE:[39m Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:00:51 [35mSTATE:[39m Build for: browserBundle type: iife: {"imports":40,"importBytes":1643099,"outputBytes":1402155,"outputFiles":"dist/human.js"}
|
||||
2021-04-24 16:00:51 [35mSTATE:[39m Build for: browserBundle type: esm: {"imports":40,"importBytes":1643099,"outputBytes":1402113,"outputFiles":"dist/human.esm.js"}
|
||||
2021-04-24 16:00:51 [36mINFO: [39m Generate types: ["src/human.ts"]
|
||||
2021-04-24 16:00:57 [36mINFO: [39m Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-04-24 16:00:57 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"]
|
||||
2021-04-24 16:03:56 [36mINFO: [39m @vladmandic/human version 1.6.1
|
||||
2021-04-24 16:03:56 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.0.0
|
||||
2021-04-24 16:03:56 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true,"sourcemap":true,"bundle":true,"metafile":true,"target":"es2018"}
|
||||
2021-04-24 16:03:56 [35mSTATE:[39m Build for: node type: tfjs: {"imports":1,"importBytes":39,"outputBytes":733,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:03:56 [35mSTATE:[39m Build for: node type: node: {"imports":40,"importBytes":542082,"outputBytes":304213,"outputFiles":"dist/human.node.js"}
|
||||
2021-04-24 16:03:56 [35mSTATE:[39m Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":43,"outputBytes":737,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: nodeGPU type: node: {"imports":40,"importBytes":542086,"outputBytes":304221,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":81,"outputBytes":783,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: nodeWASM type: node: {"imports":40,"importBytes":542132,"outputBytes":304265,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2488,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: browserNoBundle type: esm: {"imports":40,"importBytes":542743,"outputBytes":304246,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-04-24 16:03:57 [35mSTATE:[39m Build for: browserBundle type: tfjs: {"modules":1267,"moduleBytes":4085087,"imports":7,"importBytes":2488,"outputBytes":1101728,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-04-24 16:03:58 [35mSTATE:[39m Build for: browserBundle type: iife: {"imports":40,"importBytes":1643077,"outputBytes":1402155,"outputFiles":"dist/human.js"}
|
||||
2021-04-24 16:03:58 [35mSTATE:[39m Build for: browserBundle type: esm: {"imports":40,"importBytes":1643077,"outputBytes":1402113,"outputFiles":"dist/human.esm.js"}
|
||||
2021-04-24 16:03:58 [36mINFO: [39m Generate types: ["src/human.ts"]
|
||||
2021-04-24 16:04:03 [36mINFO: [39m Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-04-24 16:04:03 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"]
|
||||
|
|
|
@ -366,7 +366,7 @@ const config: Config = {
|
|||
maxDetections: 1, // maximum number of people detected in the input
|
||||
// should be set to the minimum number for performance
|
||||
// only valid for posenet as blazepose only detects single pose
|
||||
scoreThreshold: 0.3, // threshold for deciding when to remove boxes based on score
|
||||
scoreThreshold: 0.2, // threshold for deciding when to remove boxes based on score
|
||||
// in non-maximum suppression
|
||||
// only valid for posenet as blazepose only detects single pose
|
||||
nmsRadius: 20, // radius for deciding points are too close in non-maximum suppression
|
||||
|
|
|
@ -88,7 +88,7 @@ export class Human {
|
|||
/** Internal: Currently loaded models */
|
||||
models: {
|
||||
face: facemesh.MediaPipeFaceMesh | Model | null,
|
||||
posenet: posenet.PoseNet | null,
|
||||
posenet: Model | null,
|
||||
blazepose: Model | null,
|
||||
efficientpose: Model | null,
|
||||
handpose: handpose.HandPose | null,
|
||||
|
@ -444,14 +444,14 @@ export class Human {
|
|||
// run body: can be posenet or blazepose
|
||||
this.analyze('Start Body:');
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
|
||||
if (this.perf.body) delete this.perf.body;
|
||||
} else {
|
||||
this.state = 'run:body';
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
|
|
|
@ -1,78 +0,0 @@
|
|||
import * as decodeSingle from './decodeSingle';
|
||||
import * as utils from './utils';
|
||||
|
||||
const kLocalMaximumRadius = 1;
|
||||
const defaultOutputStride = 16;
|
||||
|
||||
function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores) {
|
||||
const [height, width] = scores.shape;
|
||||
let localMaximum = true;
|
||||
const yStart = Math.max(heatmapY - localMaximumRadius, 0);
|
||||
const yEnd = Math.min(heatmapY + localMaximumRadius + 1, height);
|
||||
for (let yCurrent = yStart; yCurrent < yEnd; ++yCurrent) {
|
||||
const xStart = Math.max(heatmapX - localMaximumRadius, 0);
|
||||
const xEnd = Math.min(heatmapX + localMaximumRadius + 1, width);
|
||||
for (let xCurrent = xStart; xCurrent < xEnd; ++xCurrent) {
|
||||
if (scores.get(yCurrent, xCurrent, keypointId) > score) {
|
||||
localMaximum = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!localMaximum) break;
|
||||
}
|
||||
return localMaximum;
|
||||
}
|
||||
|
||||
export function buildPartWithScoreQueue(scoreThreshold, localMaximumRadius, scores) {
|
||||
const [height, width, numKeypoints] = scores.shape;
|
||||
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
|
||||
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
|
||||
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
|
||||
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
|
||||
const score = scores.get(heatmapY, heatmapX, keypointId);
|
||||
// Only consider parts with score greater or equal to threshold as root candidates.
|
||||
if (score < scoreThreshold) continue;
|
||||
// Only consider keypoints whose score is maximum in a local window.
|
||||
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, localMaximumRadius, scores)) {
|
||||
queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return queue;
|
||||
}
|
||||
|
||||
function withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, { x, y }, keypointId) {
|
||||
return poses.some(({ keypoints }) => {
|
||||
const correspondingKeypoint = keypoints[keypointId].position;
|
||||
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
|
||||
});
|
||||
}
|
||||
|
||||
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
|
||||
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
|
||||
if (!withinNmsRadiusOfCorrespondingPoint(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
|
||||
return result;
|
||||
}, 0.0);
|
||||
return notOverlappedKeypointScores / instanceKeypoints.length;
|
||||
}
|
||||
|
||||
export function decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
|
||||
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
|
||||
const queue = buildPartWithScoreQueue(scoreThreshold, kLocalMaximumRadius, scoresBuffer);
|
||||
const squaredNmsRadius = nmsRadius ^ 2;
|
||||
// Generate at most maxDetections object instances per image in decreasing root part score order.
|
||||
while (poses.length < maxDetections && !queue.empty()) {
|
||||
// The top element in the queue is the next root candidate.
|
||||
const root = queue.dequeue();
|
||||
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
|
||||
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
|
||||
if (withinNmsRadiusOfCorrespondingPoint(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
|
||||
// Else start a new detection instance at the position of the root.
|
||||
const keypoints = decodeSingle.decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
|
||||
const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
|
||||
const box = utils.getBoundingBox(keypoints);
|
||||
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
|
||||
}
|
||||
return poses;
|
||||
}
|
|
@ -1,48 +0,0 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as keypoints from './keypoints';
|
||||
|
||||
export function getPointsConfidence(heatmapScores, heatMapCoords) {
|
||||
const numKeypoints = keypoints.count; // also in heatMapCoords.shape[0];
|
||||
const result:Array<number> = [];
|
||||
for (let keypoint = 0; keypoint < numKeypoints; keypoint++) {
|
||||
const y = heatMapCoords.get(keypoint, 0);
|
||||
const x = heatMapCoords.get(keypoint, 1);
|
||||
result.push(heatmapScores.get(y, x, keypoint));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getOffsetPoints(heatMapCoordsBuffer, outputStride, offsetsBuffer) {
|
||||
const getOffsetPoint = (y, x, keypoint) => ([
|
||||
offsetsBuffer.get(y, x, keypoint),
|
||||
offsetsBuffer.get(y, x, keypoint + keypoints.count),
|
||||
]);
|
||||
|
||||
const getOffsetVectors = () => {
|
||||
const result: Array<number[]> = [];
|
||||
for (let keypoint = 0; keypoint < keypoints.count; keypoint++) {
|
||||
const heatmapY = heatMapCoordsBuffer.get(keypoint, 0);
|
||||
const heatmapX = heatMapCoordsBuffer.get(keypoint, 1);
|
||||
result.push(getOffsetPoint(heatmapY, heatmapX, keypoint));
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
return tf.tidy(() => heatMapCoordsBuffer.toTensor().mul(tf.scalar(outputStride, 'int32')).toFloat().add(getOffsetVectors()));
|
||||
}
|
||||
|
||||
export function argmax2d(inputs) {
|
||||
const mod = (a, b) => tf.tidy(() => {
|
||||
const floored = a.div(tf.scalar(b, 'int32'));
|
||||
return a.sub(floored.mul(tf.scalar(b, 'int32')));
|
||||
});
|
||||
const [height, width, depth] = inputs.shape;
|
||||
|
||||
return tf.tidy(() => {
|
||||
const reshaped = inputs.reshape([height * width, depth]);
|
||||
const coords = reshaped.argMax(0);
|
||||
const yCoords = coords.div(tf.scalar(width, 'int32')).expandDims(1);
|
||||
const xCoords = mod(coords, width).expandDims(1);
|
||||
return tf.concat([yCoords, xCoords], 1);
|
||||
});
|
||||
}
|
|
@ -1,104 +0,0 @@
|
|||
import * as keypoints from './keypoints';
|
||||
import * as decoders from './decodeParts';
|
||||
import * as utils from './utils';
|
||||
|
||||
const parentChildrenTuples = keypoints.poseChain.map(([parentJoinName, childJoinName]) => ([keypoints.partIds[parentJoinName], keypoints.partIds[childJoinName]]));
|
||||
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
|
||||
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
|
||||
|
||||
const defaultOutputStride = 16;
|
||||
|
||||
function getDisplacement(edgeId, point, displacements) {
|
||||
const numEdges = displacements.shape[2] / 2;
|
||||
return {
|
||||
y: displacements.get(point.y, point.x, edgeId),
|
||||
x: displacements.get(point.y, point.x, numEdges + edgeId),
|
||||
};
|
||||
}
|
||||
|
||||
function getStridedIndexNearPoint(point, outputStride, height, width) {
|
||||
return {
|
||||
y: utils.clamp(Math.round(point.y / outputStride), 0, height - 1),
|
||||
x: utils.clamp(Math.round(point.x / outputStride), 0, width - 1),
|
||||
};
|
||||
}
|
||||
|
||||
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
|
||||
const [height, width] = scoresBuffer.shape;
|
||||
// Nearest neighbor interpolation for the source->target displacements.
|
||||
const sourceKeypointIndices = getStridedIndexNearPoint(sourceKeypoint.position, outputStride, height, width);
|
||||
const displacement = getDisplacement(edgeId, sourceKeypointIndices, displacements);
|
||||
const displacedPoint = utils.addVectors(sourceKeypoint.position, displacement);
|
||||
let targetKeypoint = displacedPoint;
|
||||
for (let i = 0; i < offsetRefineStep; i++) {
|
||||
const targetKeypointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width);
|
||||
const offsetPoint = utils.getOffsetPoint(targetKeypointIndices.y, targetKeypointIndices.x, targetKeypointId, offsets);
|
||||
targetKeypoint = utils.addVectors({
|
||||
x: targetKeypointIndices.x * outputStride,
|
||||
y: targetKeypointIndices.y * outputStride,
|
||||
}, { x: offsetPoint.x, y: offsetPoint.y });
|
||||
}
|
||||
const targetKeyPointIndices = getStridedIndexNearPoint(targetKeypoint, outputStride, height, width);
|
||||
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
|
||||
return { position: targetKeypoint, part: keypoints.partNames[targetKeypointId], score };
|
||||
}
|
||||
|
||||
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
|
||||
const numParts = scores.shape[2];
|
||||
const numEdges = parentToChildEdges.length;
|
||||
const instanceKeypoints = new Array(numParts);
|
||||
// Start a new detection instance at the position of the root.
|
||||
const { part: rootPart, score: rootScore } = root;
|
||||
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
|
||||
instanceKeypoints[rootPart.id] = {
|
||||
score: rootScore,
|
||||
part: keypoints.partNames[rootPart.id],
|
||||
position: rootPoint,
|
||||
};
|
||||
// Decode the part positions upwards in the tree, following the backward displacements.
|
||||
for (let edge = numEdges - 1; edge >= 0; --edge) {
|
||||
const sourceKeypointId = parentToChildEdges[edge];
|
||||
const targetKeypointId = childToParentEdges[edge];
|
||||
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
|
||||
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
|
||||
}
|
||||
}
|
||||
// Decode the part positions downwards in the tree, following the forward displacements.
|
||||
for (let edge = 0; edge < numEdges; ++edge) {
|
||||
const sourceKeypointId = childToParentEdges[edge];
|
||||
const targetKeypointId = parentToChildEdges[edge];
|
||||
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
|
||||
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
|
||||
}
|
||||
}
|
||||
return instanceKeypoints;
|
||||
}
|
||||
|
||||
export async function decodeSinglePose(heatmapScores, offsets, minScore) {
|
||||
const heatmapValues = decoders.argmax2d(heatmapScores);
|
||||
const allTensorBuffers = await Promise.all([heatmapScores.buffer(), offsets.buffer(), heatmapValues.buffer()]);
|
||||
const scoresBuffer = allTensorBuffers[0];
|
||||
const offsetsBuffer = allTensorBuffers[1];
|
||||
const heatmapValuesBuffer = allTensorBuffers[2];
|
||||
const offsetPoints = decoders.getOffsetPoints(heatmapValuesBuffer, defaultOutputStride, offsetsBuffer);
|
||||
const offsetPointsData = offsetPoints.dataSync();
|
||||
const keypointConfidence = decoders.getPointsConfidence(scoresBuffer, heatmapValuesBuffer);
|
||||
let avgScore = 0.0;
|
||||
const filteredKeypoints = keypointConfidence
|
||||
.filter((score) => score > minScore)
|
||||
.map((score, i) => {
|
||||
avgScore += score;
|
||||
return {
|
||||
position: {
|
||||
y: offsetPointsData[2 * i + 0], // offsetPointsBuffer.get(i, 0),
|
||||
x: offsetPointsData[2 * i + 1], // offsetPointsBuffer.get(i, 1),
|
||||
},
|
||||
part: keypoints.partNames[i],
|
||||
score,
|
||||
};
|
||||
});
|
||||
heatmapValues.dispose();
|
||||
offsetPoints.dispose();
|
||||
const box = utils.getBoundingBox(filteredKeypoints);
|
||||
return { keypoints: filteredKeypoints, box, score: Math.round(100 * avgScore / filteredKeypoints.length) / 100 };
|
||||
}
|
|
@ -1,61 +1,28 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as posenetModel from './posenetModel';
|
||||
import * as decodeMultiple from './decodeMultiple';
|
||||
import * as decodeSingle from './decodeSingle';
|
||||
import * as poses from './poses';
|
||||
import * as util from './utils';
|
||||
|
||||
let model;
|
||||
const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd'/* offsets */, 'MobilenetV1/heatmap_2/BiasAdd'/* heatmapScores */, 'MobilenetV1/displacement_fwd_2/BiasAdd'/* displacementFwd */, 'MobilenetV1/displacement_bwd_2/BiasAdd'/* displacementBwd */];
|
||||
|
||||
async function estimateMultiple(input, res, config, inputSize) {
|
||||
const toTensorBuffers3D = (tensors) => Promise.all(tensors.map((tensor) => tensor.buffer()));
|
||||
|
||||
return new Promise(async (resolve) => {
|
||||
const allTensorBuffers = await toTensorBuffers3D([res.heatmapScores, res.offsets, res.displacementFwd, res.displacementBwd]);
|
||||
const scoresBuffer = allTensorBuffers[0];
|
||||
const offsetsBuffer = allTensorBuffers[1];
|
||||
const displacementsFwdBuffer = allTensorBuffers[2];
|
||||
const displacementsBwdBuffer = allTensorBuffers[3];
|
||||
const poses = await decodeMultiple.decodeMultiplePoses(scoresBuffer, offsetsBuffer, displacementsFwdBuffer, displacementsBwdBuffer, config.body.nmsRadius, config.body.maxDetections, config.body.scoreThreshold);
|
||||
const scaled = util.scalePoses(poses, [input.shape[1], input.shape[2]], [inputSize, inputSize]);
|
||||
resolve(scaled);
|
||||
export async function predict(input, config) {
|
||||
const res = tf.tidy(() => {
|
||||
const resized = input.resizeBilinear([model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
const normalized = resized.toFloat().div(127.5).sub(1.0);
|
||||
const results = model.execute(normalized, poseNetOutputs);
|
||||
const results3d = results.map((y) => y.squeeze([0]));
|
||||
results3d[1] = results3d[1].sigmoid(); // apply sigmoid on scores
|
||||
return results3d;
|
||||
});
|
||||
}
|
||||
|
||||
async function estimateSingle(input, res, config, inputSize) {
|
||||
return new Promise(async (resolve) => {
|
||||
const pose = await decodeSingle.decodeSinglePose(res.heatmapScores, res.offsets, config.body.scoreThreshold);
|
||||
const scaled = util.scalePoses([pose], [input.shape[1], input.shape[2]], [inputSize, inputSize]);
|
||||
resolve(scaled);
|
||||
});
|
||||
}
|
||||
const buffers = await Promise.all(res.map((tensor) => tensor.buffer()));
|
||||
for (const t of res) t.dispose();
|
||||
|
||||
export class PoseNet {
|
||||
baseModel: any;
|
||||
inputSize: number
|
||||
constructor(baseModel) {
|
||||
this.baseModel = baseModel;
|
||||
this.inputSize = baseModel.model.inputs[0].shape[1];
|
||||
}
|
||||
const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.nmsRadius, config.body.maxDetections, config.body.scoreThreshold);
|
||||
const scaled = util.scalePoses(decoded, [input.shape[1], input.shape[2]], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
|
||||
async estimatePoses(input, config) {
|
||||
const res = this.baseModel.predict(input, config);
|
||||
|
||||
const poses = (config.body.maxDetections < 2)
|
||||
? await estimateSingle(input, res, config, this.inputSize)
|
||||
: await estimateMultiple(input, res, config, this.inputSize);
|
||||
|
||||
res.heatmapScores.dispose();
|
||||
res.offsets.dispose();
|
||||
res.displacementFwd.dispose();
|
||||
res.displacementBwd.dispose();
|
||||
|
||||
return poses;
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.baseModel.dispose();
|
||||
}
|
||||
return scaled;
|
||||
}
|
||||
|
||||
export async function load(config) {
|
||||
|
@ -64,7 +31,5 @@ export async function load(config) {
|
|||
if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath);
|
||||
else if (config.debug) log('load model:', model.modelUrl);
|
||||
} else if (config.debug) log('cached model:', model.modelUrl);
|
||||
const mobilenet = new posenetModel.BaseModel(model);
|
||||
const poseNet = new PoseNet(mobilenet);
|
||||
return poseNet;
|
||||
return model;
|
||||
}
|
||||
|
|
|
@ -1,38 +0,0 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
|
||||
const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd', 'MobilenetV1/heatmap_2/BiasAdd', 'MobilenetV1/displacement_fwd_2/BiasAdd', 'MobilenetV1/displacement_bwd_2/BiasAdd'];
|
||||
|
||||
function nameOutputResultsMobileNet(results) {
|
||||
const [offsets, heatmap, displacementFwd, displacementBwd] = results;
|
||||
return { offsets, heatmap, displacementFwd, displacementBwd };
|
||||
}
|
||||
|
||||
export class BaseModel {
|
||||
model: any;
|
||||
inputSize: number;
|
||||
constructor(model) {
|
||||
this.model = model;
|
||||
this.inputSize = model.inputs[0].shape[1];
|
||||
}
|
||||
|
||||
predict(input) {
|
||||
return tf.tidy(() => {
|
||||
const resized = input.resizeBilinear([this.inputSize, this.inputSize]);
|
||||
const normalized = resized.toFloat().div(127.5).sub(1.0);
|
||||
// const asBatch = asFloat.expandDims(0);
|
||||
const results = this.model.execute(normalized, poseNetOutputs);
|
||||
const results3d = results.map((y) => y.squeeze([0]));
|
||||
const namedResults = nameOutputResultsMobileNet(results3d);
|
||||
return {
|
||||
heatmapScores: namedResults.heatmap.sigmoid(),
|
||||
offsets: namedResults.offsets,
|
||||
displacementFwd: namedResults.displacementFwd,
|
||||
displacementBwd: namedResults.displacementBwd,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.model.dispose();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,140 @@
|
|||
import * as utils from './utils';
|
||||
import * as kpt from './keypoints';
|
||||
|
||||
const localMaximumRadius = 1;
|
||||
const defaultOutputStride = 16;
|
||||
|
||||
function traverseToTargetKeypoint(edgeId, sourceKeypoint, targetKeypointId, scoresBuffer, offsets, outputStride, displacements, offsetRefineStep = 2) {
|
||||
const getDisplacement = (point) => ({
|
||||
y: displacements.get(point.y, point.x, edgeId),
|
||||
x: displacements.get(point.y, point.x, (displacements.shape[2] / 2) + edgeId),
|
||||
});
|
||||
const getStridedIndexNearPoint = (point, height, width) => ({
|
||||
y: utils.clamp(Math.round(point.y / outputStride), 0, height - 1),
|
||||
x: utils.clamp(Math.round(point.x / outputStride), 0, width - 1),
|
||||
});
|
||||
|
||||
const [height, width] = scoresBuffer.shape;
|
||||
// Nearest neighbor interpolation for the source->target displacements.
|
||||
const sourceKeypointIndices = getStridedIndexNearPoint(sourceKeypoint.position, height, width);
|
||||
const displacement = getDisplacement(sourceKeypointIndices);
|
||||
const displacedPoint = utils.addVectors(sourceKeypoint.position, displacement);
|
||||
let targetKeypoint = displacedPoint;
|
||||
for (let i = 0; i < offsetRefineStep; i++) {
|
||||
const targetKeypointIndices = getStridedIndexNearPoint(targetKeypoint, height, width);
|
||||
const offsetPoint = utils.getOffsetPoint(targetKeypointIndices.y, targetKeypointIndices.x, targetKeypointId, offsets);
|
||||
targetKeypoint = utils.addVectors({
|
||||
x: targetKeypointIndices.x * outputStride,
|
||||
y: targetKeypointIndices.y * outputStride,
|
||||
}, { x: offsetPoint.x, y: offsetPoint.y });
|
||||
}
|
||||
const targetKeyPointIndices = getStridedIndexNearPoint(targetKeypoint, height, width);
|
||||
const score = scoresBuffer.get(targetKeyPointIndices.y, targetKeyPointIndices.x, targetKeypointId);
|
||||
return { position: targetKeypoint, part: kpt.partNames[targetKeypointId], score };
|
||||
}
|
||||
|
||||
export function decodePose(root, scores, offsets, outputStride, displacementsFwd, displacementsBwd) {
|
||||
const parentChildrenTuples = kpt.poseChain.map(([parentJoinName, childJoinName]) => ([kpt.partIds[parentJoinName], kpt.partIds[childJoinName]]));
|
||||
const parentToChildEdges = parentChildrenTuples.map(([, childJointId]) => childJointId);
|
||||
const childToParentEdges = parentChildrenTuples.map(([parentJointId]) => parentJointId);
|
||||
const numParts = scores.shape[2];
|
||||
const numEdges = parentToChildEdges.length;
|
||||
const instanceKeypoints = new Array(numParts);
|
||||
// Start a new detection instance at the position of the root.
|
||||
const { part: rootPart, score: rootScore } = root;
|
||||
const rootPoint = utils.getImageCoords(rootPart, outputStride, offsets);
|
||||
instanceKeypoints[rootPart.id] = {
|
||||
score: rootScore,
|
||||
part: kpt.partNames[rootPart.id],
|
||||
position: rootPoint,
|
||||
};
|
||||
// Decode the part positions upwards in the tree, following the backward displacements.
|
||||
for (let edge = numEdges - 1; edge >= 0; --edge) {
|
||||
const sourceKeypointId = parentToChildEdges[edge];
|
||||
const targetKeypointId = childToParentEdges[edge];
|
||||
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
|
||||
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsBwd);
|
||||
}
|
||||
}
|
||||
// Decode the part positions downwards in the tree, following the forward displacements.
|
||||
for (let edge = 0; edge < numEdges; ++edge) {
|
||||
const sourceKeypointId = childToParentEdges[edge];
|
||||
const targetKeypointId = parentToChildEdges[edge];
|
||||
if (instanceKeypoints[sourceKeypointId] && !instanceKeypoints[targetKeypointId]) {
|
||||
instanceKeypoints[targetKeypointId] = traverseToTargetKeypoint(edge, instanceKeypoints[sourceKeypointId], targetKeypointId, scores, offsets, outputStride, displacementsFwd);
|
||||
}
|
||||
}
|
||||
return instanceKeypoints;
|
||||
}
|
||||
|
||||
function scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores) {
|
||||
const [height, width] = scores.shape;
|
||||
let localMaximum = true;
|
||||
const yStart = Math.max(heatmapY - localMaximumRadius, 0);
|
||||
const yEnd = Math.min(heatmapY + localMaximumRadius + 1, height);
|
||||
for (let yCurrent = yStart; yCurrent < yEnd; ++yCurrent) {
|
||||
const xStart = Math.max(heatmapX - localMaximumRadius, 0);
|
||||
const xEnd = Math.min(heatmapX + localMaximumRadius + 1, width);
|
||||
for (let xCurrent = xStart; xCurrent < xEnd; ++xCurrent) {
|
||||
if (scores.get(yCurrent, xCurrent, keypointId) > score) {
|
||||
localMaximum = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!localMaximum) break;
|
||||
}
|
||||
return localMaximum;
|
||||
}
|
||||
|
||||
export function buildPartWithScoreQueue(scoreThreshold, scores) {
|
||||
const [height, width, numKeypoints] = scores.shape;
|
||||
const queue = new utils.MaxHeap(height * width * numKeypoints, ({ score }) => score);
|
||||
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
|
||||
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
|
||||
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
|
||||
const score = scores.get(heatmapY, heatmapX, keypointId);
|
||||
// Only consider parts with score greater or equal to threshold as root candidates.
|
||||
if (score < scoreThreshold) continue;
|
||||
// Only consider keypoints whose score is maximum in a local window.
|
||||
if (scoreIsMaximumInLocalWindow(keypointId, score, heatmapY, heatmapX, scores)) queue.enqueue({ score, part: { heatmapY, heatmapX, id: keypointId } });
|
||||
}
|
||||
}
|
||||
}
|
||||
return queue;
|
||||
}
|
||||
|
||||
function withinRadius(poses, squaredNmsRadius, { x, y }, keypointId) {
|
||||
return poses.some(({ keypoints }) => {
|
||||
const correspondingKeypoint = keypoints[keypointId].position;
|
||||
return utils.squaredDistance(y, x, correspondingKeypoint.y, correspondingKeypoint.x) <= squaredNmsRadius;
|
||||
});
|
||||
}
|
||||
|
||||
function getInstanceScore(existingPoses, squaredNmsRadius, instanceKeypoints) {
|
||||
const notOverlappedKeypointScores = instanceKeypoints.reduce((result, { position, score }, keypointId) => {
|
||||
if (!withinRadius(existingPoses, squaredNmsRadius, position, keypointId)) result += score;
|
||||
return result;
|
||||
}, 0.0);
|
||||
return notOverlappedKeypointScores / instanceKeypoints.length;
|
||||
}
|
||||
|
||||
export function decode(offsetsBuffer, scoresBuffer, displacementsFwdBuffer, displacementsBwdBuffer, nmsRadius, maxDetections, scoreThreshold) {
|
||||
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
|
||||
const queue = buildPartWithScoreQueue(scoreThreshold, scoresBuffer);
|
||||
const squaredNmsRadius = nmsRadius ** 2;
|
||||
// Generate at most maxDetections object instances per image in decreasing root part score order.
|
||||
while (poses.length < maxDetections && !queue.empty()) {
|
||||
// The top element in the queue is the next root candidate.
|
||||
const root = queue.dequeue();
|
||||
// Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance.
|
||||
const rootImageCoords = utils.getImageCoords(root.part, defaultOutputStride, offsetsBuffer);
|
||||
if (withinRadius(poses, squaredNmsRadius, rootImageCoords, root.part.id)) continue;
|
||||
// Else start a new detection instance at the position of the root.
|
||||
const allKeypoints = decodePose(root, scoresBuffer, offsetsBuffer, defaultOutputStride, displacementsFwdBuffer, displacementsBwdBuffer);
|
||||
const keypoints = allKeypoints.filter((a) => a.score > scoreThreshold);
|
||||
const score = getInstanceScore(poses, squaredNmsRadius, keypoints);
|
||||
const box = utils.getBoundingBox(keypoints);
|
||||
if (score > scoreThreshold) poses.push({ keypoints, box, score: Math.round(100 * score) / 100 });
|
||||
}
|
||||
return poses;
|
||||
}
|
|
@ -452,7 +452,7 @@
|
|||
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-class">
|
||||
<a name="models" class="tsd-anchor"></a>
|
||||
<h3>models</h3>
|
||||
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span><span class="tsd-signature-symbol"> }</span></div>
|
||||
<div class="tsd-signature tsd-kind-icon">models<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>age<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>blazepose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>efficientpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>embedding<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>emotion<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>face<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">MediaPipeFaceMesh</span><span class="tsd-signature-symbol">; </span>faceres<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>gender<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>handpose<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HandPose</span><span class="tsd-signature-symbol">; </span>iris<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol">; </span>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span><span class="tsd-signature-symbol"> }</span></div>
|
||||
<aside class="tsd-sources">
|
||||
</aside>
|
||||
<div class="tsd-comment tsd-typography">
|
||||
|
@ -497,7 +497,7 @@
|
|||
<h5>nanodet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
|
||||
</li>
|
||||
<li class="tsd-parameter">
|
||||
<h5>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">PoseNet</span></h5>
|
||||
<h5>posenet<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">null</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">Object</span></h5>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
|
|
@ -76,7 +76,7 @@ export declare class Human {
|
|||
/** Internal: Currently loaded models */
|
||||
models: {
|
||||
face: facemesh.MediaPipeFaceMesh | Model | null;
|
||||
posenet: posenet.PoseNet | null;
|
||||
posenet: Model | null;
|
||||
blazepose: Model | null;
|
||||
efficientpose: Model | null;
|
||||
handpose: handpose.HandPose | null;
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
import * as utils from './utils';
|
||||
export declare function buildPartWithScoreQueue(scoreThreshold: any, localMaximumRadius: any, scores: any): utils.MaxHeap;
|
||||
export declare function decodeMultiplePoses(scoresBuffer: any, offsetsBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, nmsRadius: any, maxDetections: any, scoreThreshold: any): {
|
||||
keypoints: any;
|
||||
box: any;
|
||||
score: number;
|
||||
}[];
|
|
@ -1,3 +0,0 @@
|
|||
export declare function getPointsConfidence(heatmapScores: any, heatMapCoords: any): number[];
|
||||
export declare function getOffsetPoints(heatMapCoordsBuffer: any, outputStride: any, offsetsBuffer: any): any;
|
||||
export declare function argmax2d(inputs: any): any;
|
|
@ -1,13 +0,0 @@
|
|||
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
|
||||
export declare function decodeSinglePose(heatmapScores: any, offsets: any, minScore: any): Promise<{
|
||||
keypoints: {
|
||||
position: {
|
||||
y: any;
|
||||
x: any;
|
||||
};
|
||||
part: string;
|
||||
score: number;
|
||||
}[];
|
||||
box: any[];
|
||||
score: number;
|
||||
}>;
|
|
@ -1,8 +1,2 @@
|
|||
export declare class PoseNet {
|
||||
baseModel: any;
|
||||
inputSize: number;
|
||||
constructor(baseModel: any);
|
||||
estimatePoses(input: any, config: any): Promise<unknown>;
|
||||
dispose(): void;
|
||||
}
|
||||
export declare function load(config: any): Promise<PoseNet>;
|
||||
export declare function predict(input: any, config: any): Promise<any>;
|
||||
export declare function load(config: any): Promise<any>;
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
export declare class BaseModel {
|
||||
model: any;
|
||||
inputSize: number;
|
||||
constructor(model: any);
|
||||
predict(input: any): any;
|
||||
dispose(): void;
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
import * as utils from './utils';
|
||||
export declare function decodePose(root: any, scores: any, offsets: any, outputStride: any, displacementsFwd: any, displacementsBwd: any): any[];
|
||||
export declare function buildPartWithScoreQueue(scoreThreshold: any, scores: any): utils.MaxHeap;
|
||||
export declare function decode(offsetsBuffer: any, scoresBuffer: any, displacementsFwdBuffer: any, displacementsBwdBuffer: any, nmsRadius: any, maxDetections: any, scoreThreshold: any): {
|
||||
keypoints: any;
|
||||
box: any;
|
||||
score: number;
|
||||
}[];
|
Loading…
Reference in New Issue