update movenet-multipose and samples
Vladimir Mandic
|
|
@ -4,16 +4,16 @@ const process = require('process');
|
|||
const log = require('@vladmandic/pilogger');
|
||||
const canvas = require('canvas');
|
||||
const tf = require('@tensorflow/tfjs-node'); // for nodejs, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
|
||||
const Human = require('../dist/human.node.js'); // this is 'const Human = require('../dist/human.node-gpu.js').default;'
|
||||
const Human = require('../../dist/human.node.js'); // this is 'const Human = require('../dist/human.node-gpu.js').default;'
|
||||
|
||||
const config = { // just enable all and leave default settings
|
||||
debug: true,
|
||||
async: false,
|
||||
cacheSensitivity: 0,
|
||||
face: { enabled: true },
|
||||
face: { enabled: true, detector: { maxDetected: 20 } },
|
||||
object: { enabled: true },
|
||||
gesture: { enabled: true },
|
||||
hand: { enabled: true, minConfidence: 0.4, detector: { modelPath: 'handtrack.json' } },
|
||||
hand: { enabled: true },
|
||||
body: { enabled: true, modelPath: 'https://vladmandic.github.io/human-models/models/movenet-multipose.json' },
|
||||
};
|
||||
|
||||
|
|
@ -5,8 +5,6 @@ Not required for normal funcioning of library
|
|||
|
||||
Samples were generated using command:
|
||||
|
||||
```shell
|
||||
node test/test-node-canvas.js samples/in/ samples/out/
|
||||
```
|
||||
> node demo/nodejs/process-folder.js samples/in/ samples/out/
|
||||
|
||||
Samples galery viewer: <https://vladmandic.github.io/human/samples/samples.html>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 44 KiB After Width: | Height: | Size: 45 KiB |
|
Before Width: | Height: | Size: 9.6 KiB After Width: | Height: | Size: 9.8 KiB |
|
Before Width: | Height: | Size: 42 KiB After Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 374 KiB After Width: | Height: | Size: 368 KiB |
|
Before Width: | Height: | Size: 142 KiB After Width: | Height: | Size: 124 KiB |
|
Before Width: | Height: | Size: 321 KiB After Width: | Height: | Size: 323 KiB |
|
Before Width: | Height: | Size: 327 KiB After Width: | Height: | Size: 309 KiB |
|
Before Width: | Height: | Size: 424 KiB After Width: | Height: | Size: 423 KiB |
|
Before Width: | Height: | Size: 223 KiB After Width: | Height: | Size: 209 KiB |
|
Before Width: | Height: | Size: 232 KiB After Width: | Height: | Size: 233 KiB |
|
Before Width: | Height: | Size: 70 KiB After Width: | Height: | Size: 73 KiB |
|
Before Width: | Height: | Size: 74 KiB After Width: | Height: | Size: 69 KiB |
|
Before Width: | Height: | Size: 127 KiB After Width: | Height: | Size: 127 KiB |
|
Before Width: | Height: | Size: 103 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 118 KiB After Width: | Height: | Size: 119 KiB |
|
Before Width: | Height: | Size: 185 KiB After Width: | Height: | Size: 174 KiB |
|
Before Width: | Height: | Size: 71 KiB After Width: | Height: | Size: 74 KiB |
|
Before Width: | Height: | Size: 50 KiB After Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 79 KiB After Width: | Height: | Size: 80 KiB |
|
Before Width: | Height: | Size: 120 KiB After Width: | Height: | Size: 115 KiB |
|
Before Width: | Height: | Size: 181 KiB After Width: | Height: | Size: 182 KiB |
|
|
@ -21,12 +21,14 @@
|
|||
::-webkit-scrollbar-track { margin: 3px; }
|
||||
.text { margin: 24px }
|
||||
.strip { display: flex; width: 100%; overflow-x: auto; overflow-y: hidden; scroll-behavior: smooth; }
|
||||
.thumb { height: 180px; margin: 2px; padding: 2px; }
|
||||
.thumb { height: 180px; margin: 2px; padding: 2px; cursor: grab; }
|
||||
.thumb:hover { filter: grayscale(1); transform: scale(1.08); transition : all 0.3s ease; }
|
||||
.image-container { margin: 24px 3px 3px 3px }
|
||||
.image-container { margin: 24px 3px 3px 3px; cursor: nwse-resize; }
|
||||
.image-zoomwidth { max-width: 94vw; }
|
||||
.image-zoomheight { max-height: 70vh; }
|
||||
.image-zoomfull { max-height: -webkit-fill-available; }
|
||||
.arrow { font-size: 2rem; font-weight: bolder; background: grey; border-radius: 4px; position: fixed; top: 140px; opacity: 65%; cursor: pointer; }
|
||||
.arrow:hover { background: white; color: grey }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -34,6 +36,8 @@
|
|||
<div id="strip" class="strip"></div>
|
||||
<div class="image-container">
|
||||
<img id="image" src="data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=" alt="" class="image-zoomwidth" />
|
||||
<div class="arrow" id="arrow-left" style="left: 10px"><</div>
|
||||
<div class="arrow" id="arrow-right" style="right: 10px">></div>
|
||||
</div>
|
||||
<script>
|
||||
const samples = [
|
||||
|
|
@ -54,13 +58,14 @@
|
|||
else if (image.classList.contains('image-zoomfull')) image.className = 'image-zoomwidth';
|
||||
});
|
||||
const strip = document.getElementById('strip');
|
||||
document.getElementById('arrow-left').addEventListener('click', () => strip.scrollLeft -= strip.offsetWidth);
|
||||
document.getElementById('arrow-right').addEventListener('click', () => strip.scrollLeft += strip.offsetWidth);
|
||||
for (const sample of samples) {
|
||||
const el = document.createElement('img');
|
||||
el.className = 'thumb';
|
||||
el.src = el.title = el.alt = `/samples/in/${sample}`;
|
||||
el.addEventListener('click', (evt) => {
|
||||
image.src = image.alt = image.title = el.src.replace('/in/', '/out/');
|
||||
console.log(evt);
|
||||
strip.scrollLeft = evt.target.offsetLeft - window.innerWidth / 2 + evt.target.offsetWidth / 2;
|
||||
});
|
||||
strip.appendChild(el);
|
||||
|
|
|
|||
|
|
@ -38,6 +38,26 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
function createBox(points): [Box, Box] {
|
||||
const x = points.map((a) => a.position[0]);
|
||||
const y = points.map((a) => a.position[1]);
|
||||
const box: Box = [
|
||||
Math.min(...x),
|
||||
Math.min(...y),
|
||||
Math.max(...x) - Math.min(...x),
|
||||
Math.max(...y) - Math.min(...y),
|
||||
];
|
||||
const xRaw = points.map((a) => a.positionRaw[0]);
|
||||
const yRaw = points.map((a) => a.positionRaw[1]);
|
||||
const boxRaw: Box = [
|
||||
Math.min(...xRaw),
|
||||
Math.min(...yRaw),
|
||||
Math.max(...xRaw) - Math.min(...xRaw),
|
||||
Math.max(...yRaw) - Math.min(...yRaw),
|
||||
];
|
||||
return [box, boxRaw];
|
||||
}
|
||||
|
||||
async function parseSinglePose(res, config, image, inputBox) {
|
||||
const kpt = res[0][0];
|
||||
keypoints.length = 0;
|
||||
|
|
@ -61,23 +81,8 @@ async function parseSinglePose(res, config, image, inputBox) {
|
|||
}
|
||||
}
|
||||
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
|
||||
const x = keypoints.map((a) => a.position[0]);
|
||||
const y = keypoints.map((a) => a.position[1]);
|
||||
const box: Box = [
|
||||
Math.min(...x),
|
||||
Math.min(...y),
|
||||
Math.max(...x) - Math.min(...x),
|
||||
Math.max(...y) - Math.min(...y),
|
||||
];
|
||||
const xRaw = keypoints.map((a) => a.positionRaw[0]);
|
||||
const yRaw = keypoints.map((a) => a.positionRaw[1]);
|
||||
const boxRaw: Box = [
|
||||
Math.min(...xRaw),
|
||||
Math.min(...yRaw),
|
||||
Math.max(...xRaw) - Math.min(...xRaw),
|
||||
Math.max(...yRaw) - Math.min(...yRaw),
|
||||
];
|
||||
const bodies: Array<Body> = [];
|
||||
const [box, boxRaw] = createBox(keypoints);
|
||||
bodies.push({ id: 0, score, box, boxRaw, keypoints });
|
||||
return bodies;
|
||||
}
|
||||
|
|
@ -86,39 +91,36 @@ async function parseMultiPose(res, config, image, inputBox) {
|
|||
const bodies: Array<Body> = [];
|
||||
for (let id = 0; id < res[0].length; id++) {
|
||||
const kpt = res[0][id];
|
||||
const score = Math.round(100 * kpt[51 + 4]) / 100;
|
||||
// eslint-disable-next-line no-continue
|
||||
if (score < config.body.minConfidence) continue;
|
||||
keypoints.length = 0;
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const partScore = Math.round(100 * kpt[3 * i + 2]) / 100;
|
||||
if (partScore > config.body.minConfidence) {
|
||||
const positionRaw: Point = [
|
||||
(inputBox[3] - inputBox[1]) * kpt[3 * i + 1] + inputBox[1],
|
||||
(inputBox[2] - inputBox[0]) * kpt[3 * i + 0] + inputBox[0],
|
||||
];
|
||||
keypoints.push({
|
||||
part: bodyParts[i],
|
||||
score: partScore,
|
||||
positionRaw,
|
||||
position: [Math.trunc(positionRaw[0] * (image.shape[2] || 0)), Math.trunc(positionRaw[0] * (image.shape[1] || 0))],
|
||||
});
|
||||
const totalScore = Math.round(100 * kpt[51 + 4]) / 100;
|
||||
if (totalScore > config.body.minConfidence) {
|
||||
keypoints.length = 0;
|
||||
for (let i = 0; i < 17; i++) {
|
||||
const score = kpt[3 * i + 2];
|
||||
if (score > config.body.minConfidence) {
|
||||
const positionRaw: Point = [
|
||||
(inputBox[3] - inputBox[1]) * kpt[3 * i + 1] + inputBox[1],
|
||||
(inputBox[2] - inputBox[0]) * kpt[3 * i + 0] + inputBox[0],
|
||||
];
|
||||
keypoints.push({
|
||||
part: bodyParts[i],
|
||||
score: Math.round(100 * score) / 100,
|
||||
positionRaw,
|
||||
position: [
|
||||
Math.round((image.shape[2] || 0) * positionRaw[0]),
|
||||
Math.round((image.shape[1] || 0) * positionRaw[1]),
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
// const [box, boxRaw] = createBox(keypoints);
|
||||
// movenet-multipose has built-in box details
|
||||
const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
|
||||
const box: Box = [Math.trunc(boxRaw[0] * (image.shape[2] || 0)), Math.trunc(boxRaw[1] * (image.shape[1] || 0)), Math.trunc(boxRaw[2] * (image.shape[2] || 0)), Math.trunc(boxRaw[3] * (image.shape[1] || 0))];
|
||||
bodies.push({ id, score: totalScore, boxRaw, box, keypoints: [...keypoints] });
|
||||
}
|
||||
const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
|
||||
bodies.push({
|
||||
id,
|
||||
score,
|
||||
boxRaw,
|
||||
box: [
|
||||
Math.trunc(boxRaw[0] * (image.shape[2] || 0)),
|
||||
Math.trunc(boxRaw[1] * (image.shape[1] || 0)),
|
||||
Math.trunc(boxRaw[2] * (image.shape[2] || 0)),
|
||||
Math.trunc(boxRaw[3] * (image.shape[1] || 0)),
|
||||
],
|
||||
keypoints: [...keypoints],
|
||||
});
|
||||
}
|
||||
bodies.sort((a, b) => b.score - a.score);
|
||||
if (bodies.length > config.body.maxDetected) bodies.length = config.body.maxDetected;
|
||||
return bodies;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,6 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
|
|||
tf.dispose(coordsReshaped);
|
||||
if (faceConfidence < (config.face.detector?.minConfidence || 1)) {
|
||||
box.confidence = faceConfidence; // reset confidence of cached box
|
||||
tf.dispose(face.tensor);
|
||||
} else {
|
||||
if (config.face.iris?.enabled) rawCoords = await iris.augmentIris(rawCoords, face.tensor, config, inputSize); // augment results with iris
|
||||
face.mesh = util.transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize); // get processed mesh
|
||||
|
|
|
|||