mirror of https://github.com/vladmandic/human
update hand model
Vladimir Mandic
parent
1c0499d797
commit
a61b19ac0c
|
|
@ -113,6 +113,7 @@ export default {
|
|||
scoreThreshold: 0.8, // threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace
|
||||
maxHands: 1, // maximum number of hands detected in the input, should be set to the minimum number for performance
|
||||
landmarks: true, // detect hand landmarks or just hand boundary box
|
||||
detector: {
|
||||
modelPath: '../models/handdetect.json',
|
||||
},
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ const ui = {
|
|||
maxFrames: 10,
|
||||
modelsPreload: true,
|
||||
modelsWarmup: true,
|
||||
menuWidth: 0,
|
||||
menuHeight: 0,
|
||||
camera: {},
|
||||
fps: [],
|
||||
};
|
||||
|
||||
// global variables
|
||||
|
|
@ -34,8 +38,6 @@ let menu;
|
|||
let menuFX;
|
||||
let worker;
|
||||
let timeStamp;
|
||||
let camera = {};
|
||||
const fps = [];
|
||||
|
||||
// helper function: translates json to human readable string
|
||||
function str(...msg) {
|
||||
|
|
@ -62,17 +64,22 @@ const status = (msg) => {
|
|||
// draws processed results and starts processing of a next frame
|
||||
function drawResults(input, result, canvas) {
|
||||
// update fps data
|
||||
fps.push(1000 / (performance.now() - timeStamp));
|
||||
if (fps.length > ui.maxFrames) fps.shift();
|
||||
const elapsed = performance.now() - timeStamp;
|
||||
ui.fps.push(1000 / elapsed);
|
||||
if (ui.fps.length > ui.maxFrames) ui.fps.shift();
|
||||
|
||||
// enable for continous performance monitoring
|
||||
// console.log(result.performance);
|
||||
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
if (input.srcObject) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop before we even draw results
|
||||
|
||||
// immediate loop before we even draw results, but limit frame rate to 30
|
||||
if (input.srcObject) {
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
if (elapsed > 33) requestAnimationFrame(() => runHumanDetect(input, canvas));
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
else setTimeout(() => runHumanDetect(input, canvas), 33 - elapsed);
|
||||
}
|
||||
// draw fps chart
|
||||
menu.updateChart('FPS', fps);
|
||||
menu.updateChart('FPS', ui.fps);
|
||||
// draw image from video
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseBackground;
|
||||
|
|
@ -94,9 +101,9 @@ function drawResults(input, result, canvas) {
|
|||
const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : '';
|
||||
const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`;
|
||||
const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : '';
|
||||
const avg = Math.trunc(10 * fps.reduce((a, b) => a + b) / fps.length) / 10;
|
||||
const avg = Math.trunc(10 * ui.fps.reduce((a, b) => a + b) / ui.fps.length) / 10;
|
||||
document.getElementById('log').innerText = `
|
||||
video: ${camera.name} | facing: ${camera.facing} | resolution: ${camera.width} x ${camera.height} ${processing}
|
||||
video: ${ui.camera.name} | facing: ${ui.camera.facing} | resolution: ${ui.camera.width} x ${ui.camera.height} ${processing}
|
||||
backend: ${human.tf.getBackend()} | ${memory}
|
||||
performance: ${str(result.performance)} FPS:${avg}
|
||||
`;
|
||||
|
|
@ -147,7 +154,7 @@ async function setupCamera() {
|
|||
const track = stream.getVideoTracks()[0];
|
||||
const settings = track.getSettings();
|
||||
log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
|
||||
camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
|
||||
ui.camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
|
||||
return new Promise((resolve) => {
|
||||
video.onloadeddata = async () => {
|
||||
video.width = video.videoWidth;
|
||||
|
|
@ -156,6 +163,8 @@ async function setupCamera() {
|
|||
canvas.height = video.height;
|
||||
canvas.style.width = canvas.width > canvas.height ? '100vw' : '';
|
||||
canvas.style.height = canvas.width > canvas.height ? '' : '100vh';
|
||||
ui.menuWidth.input.setAttribute('value', video.width);
|
||||
ui.menuHeight.input.setAttribute('value', video.height);
|
||||
// silly font resizing for paint-on-canvas since viewport can be zoomed
|
||||
const size = 14 + (6 * canvas.width / window.innerWidth);
|
||||
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`);
|
||||
|
|
@ -351,8 +360,8 @@ function setupMenu() {
|
|||
menuFX.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
|
||||
menuFX.addLabel('Image Processing');
|
||||
menuFX.addBool('Enabled', human.config.filter, 'enabled');
|
||||
menuFX.addRange('Image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val));
|
||||
menuFX.addRange('Image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val));
|
||||
ui.menuWidth = menuFX.addRange('Image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val));
|
||||
ui.menuHeight = menuFX.addRange('Image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val));
|
||||
menuFX.addRange('Brightness', human.config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => human.config.filter.brightness = parseFloat(val));
|
||||
menuFX.addRange('Contrast', human.config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => human.config.filter.contrast = parseFloat(val));
|
||||
menuFX.addRange('Sharpness', human.config.filter, 'sharpness', 0, 1.0, 0.05, (val) => human.config.filter.sharpness = parseFloat(val));
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ class Menu {
|
|||
evt.target.setAttribute('value', evt.target.value);
|
||||
if (callback) callback(evt.target.value);
|
||||
});
|
||||
el.input = el.children[0];
|
||||
return el;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ const path = require('path');
|
|||
const dayjs = require('dayjs');
|
||||
const simpleGit = require('simple-git/promise');
|
||||
const logger = require('@vladmandic/pilogger');
|
||||
const app = require('./package.json');
|
||||
const app = require('../package.json');
|
||||
|
||||
const git = simpleGit();
|
||||
|
||||
|
|
@ -45,5 +45,5 @@ async function update(f) {
|
|||
exports.update = update;
|
||||
|
||||
if (!module.parent) {
|
||||
update('wiki/Change-Log.md');
|
||||
update('../wiki/Change-Log.md');
|
||||
}
|
||||
|
|
@ -26,9 +26,9 @@ const log = require('@vladmandic/pilogger');
|
|||
const options = {
|
||||
// key: fs.readFileSync('/home/vlado/dev/piproxy/cert/private.pem'),
|
||||
// cert: fs.readFileSync('/home/vlado/dev/piproxy/cert/fullchain.pem'),
|
||||
key: fs.readFileSync('./dev-server.key'),
|
||||
cert: fs.readFileSync('./dev-server.crt'),
|
||||
root: '.',
|
||||
key: fs.readFileSync('dev-server/dev-server.key'),
|
||||
cert: fs.readFileSync('dev-server/dev-server.crt'),
|
||||
root: '..',
|
||||
default: 'demo/index.html',
|
||||
port: 8000,
|
||||
monitor: ['package.json', 'config.js', 'demo', 'src'],
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"demo/browser.js": {
|
||||
"bytes": 17856,
|
||||
"bytes": 18278,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/human.esm.js"
|
||||
|
|
@ -19,11 +19,11 @@
|
|||
"imports": []
|
||||
},
|
||||
"demo/menu.js": {
|
||||
"bytes": 12676,
|
||||
"bytes": 12707,
|
||||
"imports": []
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"bytes": 1278200,
|
||||
"bytes": 3196462,
|
||||
"imports": []
|
||||
}
|
||||
},
|
||||
|
|
@ -31,28 +31,25 @@
|
|||
"dist/demo-browser-index.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 5532275
|
||||
"bytes": 5560779
|
||||
},
|
||||
"dist/demo-browser-index.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 1664606
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 8716
|
||||
"bytesInOutput": 3194325
|
||||
},
|
||||
"demo/draw.js": {
|
||||
"bytesInOutput": 7451
|
||||
"bytesInOutput": 7453
|
||||
},
|
||||
"demo/menu.js": {
|
||||
"bytesInOutput": 12678
|
||||
"bytesInOutput": 12709
|
||||
},
|
||||
"demo/browser.js": {
|
||||
"bytesInOutput": 15855
|
||||
"bytesInOutput": 16220
|
||||
}
|
||||
},
|
||||
"bytes": 1709428
|
||||
"bytes": 3230829
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -5,7 +5,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"package.json": {
|
||||
"bytes": 3374,
|
||||
"bytes": 3417,
|
||||
"imports": []
|
||||
},
|
||||
"src/age/age.js": {
|
||||
|
|
@ -406,7 +406,7 @@
|
|||
"bytesInOutput": 1325
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 3004
|
||||
"bytesInOutput": 3047
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7201
|
||||
|
|
@ -415,7 +415,7 @@
|
|||
"bytesInOutput": 0
|
||||
}
|
||||
},
|
||||
"bytes": 216530
|
||||
"bytes": 216573
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"config.js": {
|
||||
"bytes": 7664,
|
||||
"bytes": 7744,
|
||||
"imports": []
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
|
||||
|
|
@ -149,7 +149,7 @@
|
|||
]
|
||||
},
|
||||
"package.json": {
|
||||
"bytes": 3374,
|
||||
"bytes": 3417,
|
||||
"imports": []
|
||||
},
|
||||
"src/age/age.js": {
|
||||
|
|
@ -379,7 +379,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"bytes": 3192,
|
||||
"bytes": 3220,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -387,7 +387,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"bytes": 4220,
|
||||
"bytes": 4229,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -398,7 +398,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 8214,
|
||||
"bytes": 7445,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -412,7 +412,7 @@
|
|||
]
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytes": 3095,
|
||||
"bytes": 3029,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -513,178 +513,178 @@
|
|||
"dist/human.esm.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 5418726
|
||||
"bytes": 5609744
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"empty:/home/vlado/dev/human/node_modules/node-fetch/browser.js": {
|
||||
"bytesInOutput": 13
|
||||
"bytesInOutput": 45
|
||||
},
|
||||
"empty:util": {
|
||||
"bytesInOutput": 13
|
||||
"bytesInOutput": 42
|
||||
},
|
||||
"empty:crypto": {
|
||||
"bytesInOutput": 13
|
||||
"bytesInOutput": 44
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-core/dist/tf-core.node.js": {
|
||||
"bytesInOutput": 295162
|
||||
"bytesInOutput": 1010341
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-layers/dist/tf-layers.node.js": {
|
||||
"bytesInOutput": 238778
|
||||
"bytesInOutput": 514491
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-converter/dist/tf-converter.node.js": {
|
||||
"bytesInOutput": 115231
|
||||
"bytesInOutput": 258962
|
||||
},
|
||||
"empty:/home/vlado/dev/human/node_modules/string_decoder/lib/string_decoder.js": {
|
||||
"bytesInOutput": 13
|
||||
"bytesInOutput": 52
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-data/dist/tf-data.node.js": {
|
||||
"bytesInOutput": 52364
|
||||
"bytesInOutput": 129585
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/alea.js": {
|
||||
"bytesInOutput": 990
|
||||
"bytesInOutput": 2112
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor128.js": {
|
||||
"bytesInOutput": 755
|
||||
"bytesInOutput": 1699
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorwow.js": {
|
||||
"bytesInOutput": 845
|
||||
"bytesInOutput": 1897
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorshift7.js": {
|
||||
"bytesInOutput": 1001
|
||||
"bytesInOutput": 2307
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor4096.js": {
|
||||
"bytesInOutput": 1164
|
||||
"bytesInOutput": 2742
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/tychei.js": {
|
||||
"bytesInOutput": 880
|
||||
"bytesInOutput": 1940
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/seedrandom.js": {
|
||||
"bytesInOutput": 1614
|
||||
"bytesInOutput": 4019
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/index.js": {
|
||||
"bytesInOutput": 171
|
||||
"bytesInOutput": 458
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
|
||||
"bytesInOutput": 82512
|
||||
"bytesInOutput": 272412
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-webgl/dist/tf-backend-webgl.node.js": {
|
||||
"bytesInOutput": 261415
|
||||
"bytesInOutput": 561667
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs/dist/tf.node.js": {
|
||||
"bytesInOutput": 760
|
||||
"bytesInOutput": 3025
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 3091
|
||||
"bytesInOutput": 7099
|
||||
},
|
||||
"src/face/keypoints.js": {
|
||||
"bytesInOutput": 1946
|
||||
"bytesInOutput": 2768
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"bytesInOutput": 1006
|
||||
"bytesInOutput": 2070
|
||||
},
|
||||
"src/face/util.js": {
|
||||
"bytesInOutput": 1190
|
||||
"bytesInOutput": 3017
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 5656
|
||||
"bytesInOutput": 13567
|
||||
},
|
||||
"src/face/uvcoords.js": {
|
||||
"bytesInOutput": 16786
|
||||
"bytesInOutput": 20584
|
||||
},
|
||||
"src/face/triangulation.js": {
|
||||
"bytesInOutput": 9991
|
||||
"bytesInOutput": 23309
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"bytesInOutput": 1286
|
||||
"bytesInOutput": 2590
|
||||
},
|
||||
"src/profile.js": {
|
||||
"bytesInOutput": 620
|
||||
"bytesInOutput": 1092
|
||||
},
|
||||
"src/age/age.js": {
|
||||
"bytesInOutput": 972
|
||||
"bytesInOutput": 1843
|
||||
},
|
||||
"src/gender/gender.js": {
|
||||
"bytesInOutput": 1471
|
||||
"bytesInOutput": 2996
|
||||
},
|
||||
"src/emotion/emotion.js": {
|
||||
"bytesInOutput": 1428
|
||||
"bytesInOutput": 2715
|
||||
},
|
||||
"src/body/modelBase.js": {
|
||||
"bytesInOutput": 433
|
||||
"bytesInOutput": 900
|
||||
},
|
||||
"src/body/modelMobileNet.js": {
|
||||
"bytesInOutput": 245
|
||||
"bytesInOutput": 494
|
||||
},
|
||||
"src/body/heapSort.js": {
|
||||
"bytesInOutput": 1042
|
||||
"bytesInOutput": 1637
|
||||
},
|
||||
"src/body/buildParts.js": {
|
||||
"bytesInOutput": 547
|
||||
"bytesInOutput": 1752
|
||||
},
|
||||
"src/body/keypoints.js": {
|
||||
"bytesInOutput": 1633
|
||||
"bytesInOutput": 2277
|
||||
},
|
||||
"src/body/vectors.js": {
|
||||
"bytesInOutput": 616
|
||||
"bytesInOutput": 1408
|
||||
},
|
||||
"src/body/decodePose.js": {
|
||||
"bytesInOutput": 1024
|
||||
"bytesInOutput": 3773
|
||||
},
|
||||
"src/body/decodeMultiple.js": {
|
||||
"bytesInOutput": 604
|
||||
"bytesInOutput": 1990
|
||||
},
|
||||
"src/body/util.js": {
|
||||
"bytesInOutput": 1062
|
||||
"bytesInOutput": 2398
|
||||
},
|
||||
"src/body/modelPoseNet.js": {
|
||||
"bytesInOutput": 916
|
||||
"bytesInOutput": 2100
|
||||
},
|
||||
"src/body/posenet.js": {
|
||||
"bytesInOutput": 474
|
||||
"bytesInOutput": 903
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"bytesInOutput": 1398
|
||||
"bytesInOutput": 3583
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"bytesInOutput": 1754
|
||||
"bytesInOutput": 4485
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"bytesInOutput": 1005
|
||||
"bytesInOutput": 3419
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 2876
|
||||
"bytesInOutput": 7278
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127001
|
||||
"bytesInOutput": 256590
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytesInOutput": 1263
|
||||
"bytesInOutput": 3058
|
||||
},
|
||||
"src/gesture.js": {
|
||||
"bytesInOutput": 1220
|
||||
"bytesInOutput": 2270
|
||||
},
|
||||
"src/imagefx.js": {
|
||||
"bytesInOutput": 11014
|
||||
"bytesInOutput": 20097
|
||||
},
|
||||
"src/image.js": {
|
||||
"bytesInOutput": 2365
|
||||
"bytesInOutput": 4482
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1326
|
||||
"bytesInOutput": 2299
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 3005
|
||||
"bytesInOutput": 3561
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7219
|
||||
"bytesInOutput": 11575
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 0
|
||||
}
|
||||
},
|
||||
"bytes": 1278200
|
||||
"bytes": 3196462
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -149,7 +149,7 @@
|
|||
]
|
||||
},
|
||||
"package.json": {
|
||||
"bytes": 3374,
|
||||
"bytes": 3417,
|
||||
"imports": []
|
||||
},
|
||||
"src/age/age.js": {
|
||||
|
|
@ -675,13 +675,13 @@
|
|||
"bytesInOutput": 1326
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 3004
|
||||
"bytesInOutput": 3047
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 7257
|
||||
}
|
||||
},
|
||||
"bytes": 1278245
|
||||
"bytes": 1278288
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -5,7 +5,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"package.json": {
|
||||
"bytes": 3374,
|
||||
"bytes": 3417,
|
||||
"imports": []
|
||||
},
|
||||
"src/age/age.js": {
|
||||
|
|
@ -406,7 +406,7 @@
|
|||
"bytesInOutput": 1324
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 3004
|
||||
"bytesInOutput": 3047
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 28
|
||||
|
|
@ -415,7 +415,7 @@
|
|||
"bytesInOutput": 7201
|
||||
}
|
||||
},
|
||||
"bytes": 216537
|
||||
"bytes": 216580
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,16 +41,16 @@
|
|||
"scripts": {
|
||||
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",
|
||||
"lint": "eslint src/*.js demo/*.js",
|
||||
"dev": "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server.js",
|
||||
"dev": "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server/dev-server.js",
|
||||
"changelog": "node dev-server/changelog.js",
|
||||
"build-iife": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=iife --external:fs --global-name=Human --metafile=dist/human.json --outfile=dist/human.js src/human.js",
|
||||
"build-esm-bundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js",
|
||||
"build-esm-nobundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js",
|
||||
"build-node": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --metafile=dist/human.node.json --outfile=dist/human.node.js src/human.js",
|
||||
"build-node-nobundle": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --external:@tensorflow --metafile=dist/human.node.json --outfile=dist/human.node-nobundle.js src/human.js",
|
||||
"build-demo": "esbuild --bundle --log-level=error --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/demo-browser-index.json --outfile=dist/demo-browser-index.js demo/browser.js",
|
||||
"build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo",
|
||||
"update": "npm update --depth 20 --force && npm dedupe && npm prune && npm audit",
|
||||
"changelog": "node changelog.js"
|
||||
"build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo && npm run changelog",
|
||||
"update": "npm update --depth 20 --force && npm dedupe && npm prune && npm audit"
|
||||
},
|
||||
"keywords": [
|
||||
"tensorflowjs",
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ function scaleBoxCoordinates(box, factor) {
|
|||
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
|
||||
return scaledCoord;
|
||||
});
|
||||
return { startPoint, endPoint, palmLandmarks };
|
||||
return { startPoint, endPoint, palmLandmarks, confidence: box.confidence };
|
||||
}
|
||||
function enlargeBox(box, factor = 1.5) {
|
||||
const center = getBoxCenter(box);
|
||||
|
|
|
|||
|
|
@ -49,29 +49,28 @@ class HandDetector {
|
|||
async getBoxes(input, config) {
|
||||
const batched = this.model.predict(input);
|
||||
const predictions = batched.squeeze();
|
||||
batched.dispose();
|
||||
const scores = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze());
|
||||
// const scoresVal = scores.dataSync(); // scoresVal[boxIndex] is box confidence
|
||||
const scoresVal = scores.dataSync();
|
||||
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
|
||||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
const boxesWithHandsT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
|
||||
const boxesWithHands = boxesWithHandsT.arraySync();
|
||||
const toDispose = [
|
||||
batched,
|
||||
boxesWithHandsT,
|
||||
predictions,
|
||||
boxes,
|
||||
rawBoxes,
|
||||
scores,
|
||||
];
|
||||
rawBoxes.dispose();
|
||||
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
|
||||
const filtered = filteredT.arraySync();
|
||||
scores.dispose();
|
||||
filteredT.dispose();
|
||||
const hands = [];
|
||||
for (const boxIndex of boxesWithHands) {
|
||||
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf.slice(predictions, [boxIndex, 5], [1, 14]);
|
||||
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2]));
|
||||
rawPalmLandmarks.dispose();
|
||||
hands.push({ box: matchingBox, palmLandmarks });
|
||||
for (const boxIndex of filtered) {
|
||||
if (scoresVal[boxIndex] >= config.minConfidence) {
|
||||
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf.slice(predictions, [boxIndex, 5], [1, 14]);
|
||||
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2]));
|
||||
rawPalmLandmarks.dispose();
|
||||
hands.push({ box: matchingBox, palmLandmarks, confidence: scoresVal[boxIndex] });
|
||||
}
|
||||
}
|
||||
toDispose.forEach((tensor) => tensor.dispose());
|
||||
predictions.dispose();
|
||||
boxes.dispose();
|
||||
return hands;
|
||||
}
|
||||
|
||||
|
|
@ -84,13 +83,13 @@ class HandDetector {
|
|||
if (!predictions || predictions.length === 0) return null;
|
||||
const hands = [];
|
||||
for (const prediction of predictions) {
|
||||
const boundingBoxes = prediction.box.dataSync();
|
||||
const startPoint = boundingBoxes.slice(0, 2);
|
||||
const endPoint = boundingBoxes.slice(2, 4);
|
||||
const boxes = prediction.box.dataSync();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = prediction.palmLandmarks.arraySync();
|
||||
prediction.box.dispose();
|
||||
prediction.palmLandmarks.dispose();
|
||||
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / config.inputSize, inputHeight / config.inputSize]));
|
||||
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / config.inputSize, inputHeight / config.inputSize]));
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ const tf = require('@tensorflow/tfjs');
|
|||
const box = require('./box');
|
||||
const util = require('./util');
|
||||
|
||||
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8;
|
||||
const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
|
||||
const PALM_BOX_ENLARGE_FACTOR = 3;
|
||||
const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
|
||||
|
|
@ -87,68 +86,75 @@ class HandPipeline {
|
|||
async estimateHands(image, config) {
|
||||
this.skipped++;
|
||||
let useFreshBox = false;
|
||||
// run new detector every skipFrames
|
||||
const boxes = (this.skipped > config.skipFrames)
|
||||
? await this.boxDetector.estimateHandBounds(image, config) : null;
|
||||
|
||||
// run new detector every skipFrames unless we only want box to start with
|
||||
let boxes;
|
||||
if ((this.skipped > config.skipFrames) || !config.landmarks) {
|
||||
boxes = await this.boxDetector.estimateHandBounds(image, config);
|
||||
this.skipped = 0;
|
||||
}
|
||||
|
||||
// if detector result count doesn't match current working set, use it to reset current working set
|
||||
if (boxes && (boxes.length !== this.detectedHands) && (this.detectedHands !== config.maxHands)) {
|
||||
// console.log(this.skipped, config.maxHands, this.detectedHands, this.storedBoxes.length, boxes.length);
|
||||
if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.maxHands) || !config.landmarks)) {
|
||||
this.storedBoxes = [];
|
||||
this.detectedHands = 0;
|
||||
for (const possible of boxes) this.storedBoxes.push(possible);
|
||||
if (this.storedBoxes.length > 0) useFreshBox = true;
|
||||
this.skipped = 0;
|
||||
}
|
||||
const hands = [];
|
||||
// console.log(`skipped: ${this.skipped} max: ${config.maxHands} detected: ${this.detectedHands} stored: ${this.storedBoxes.length} new: ${boxes?.length}`);
|
||||
|
||||
// go through working set of boxes
|
||||
for (const i in this.storedBoxes) {
|
||||
const currentBox = this.storedBoxes[i];
|
||||
if (!currentBox) continue;
|
||||
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
|
||||
const palmCenter = box.getBoxCenter(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
|
||||
const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
|
||||
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
const handImage = croppedInput.div(255);
|
||||
croppedInput.dispose();
|
||||
rotatedImage.dispose();
|
||||
const [confidence, keypoints] = await this.meshDetector.predict(handImage);
|
||||
handImage.dispose();
|
||||
const confidenceValue = confidence.dataSync()[0];
|
||||
confidence.dispose();
|
||||
if (confidenceValue >= config.minConfidence) {
|
||||
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
|
||||
const rawCoords = keypointsReshaped.arraySync();
|
||||
if (config.landmarks) {
|
||||
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
|
||||
const palmCenter = box.getBoxCenter(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
|
||||
const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
|
||||
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
const handImage = croppedInput.div(255);
|
||||
croppedInput.dispose();
|
||||
rotatedImage.dispose();
|
||||
const [confidence, keypoints] = await this.meshDetector.predict(handImage);
|
||||
handImage.dispose();
|
||||
const confidenceValue = confidence.dataSync()[0];
|
||||
confidence.dispose();
|
||||
if (confidenceValue >= config.minConfidence) {
|
||||
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
|
||||
const rawCoords = keypointsReshaped.arraySync();
|
||||
keypoints.dispose();
|
||||
keypointsReshaped.dispose();
|
||||
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
|
||||
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
|
||||
this.storedBoxes[i] = nextBoundingBox;
|
||||
const result = {
|
||||
landmarks: coords,
|
||||
confidence: confidenceValue,
|
||||
box: {
|
||||
topLeft: nextBoundingBox.startPoint,
|
||||
bottomRight: nextBoundingBox.endPoint,
|
||||
},
|
||||
};
|
||||
hands.push(result);
|
||||
} else {
|
||||
this.storedBoxes[i] = null;
|
||||
}
|
||||
keypoints.dispose();
|
||||
keypointsReshaped.dispose();
|
||||
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
|
||||
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
|
||||
this.updateStoredBoxes(nextBoundingBox, i);
|
||||
const result = {
|
||||
landmarks: coords,
|
||||
handInViewConfidence: confidenceValue,
|
||||
boundingBox: {
|
||||
topLeft: nextBoundingBox.startPoint,
|
||||
bottomRight: nextBoundingBox.endPoint,
|
||||
},
|
||||
};
|
||||
hands.push(result);
|
||||
} else {
|
||||
this.updateStoredBoxes(null, i);
|
||||
/*
|
||||
const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
|
||||
const result = {
|
||||
handInViewConfidence: confidenceValue,
|
||||
boundingBox: {
|
||||
topLeft: currentBox.startPoint,
|
||||
bottomRight: currentBox.endPoint,
|
||||
confidence: currentBox.confidence,
|
||||
box: {
|
||||
topLeft: enlarged.startPoint,
|
||||
bottomRight: enlarged.endPoint,
|
||||
},
|
||||
};
|
||||
hands.push(result);
|
||||
*/
|
||||
}
|
||||
keypoints.dispose();
|
||||
}
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a !== null);
|
||||
this.detectedHands = hands.length;
|
||||
|
|
@ -163,26 +169,6 @@ class HandPipeline {
|
|||
const endPoint = [Math.max(...xs), Math.max(...ys)];
|
||||
return { startPoint, endPoint };
|
||||
}
|
||||
|
||||
updateStoredBoxes(newBox, i) {
|
||||
const previousBox = this.storedBoxes[i];
|
||||
let iou = 0;
|
||||
if (newBox && previousBox && previousBox.startPoint) {
|
||||
const [boxStartX, boxStartY] = newBox.startPoint;
|
||||
const [boxEndX, boxEndY] = newBox.endPoint;
|
||||
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;
|
||||
const [previousBoxEndX, previousBoxEndY] = previousBox.endPoint;
|
||||
const xStartMax = Math.max(boxStartX, previousBoxStartX);
|
||||
const yStartMax = Math.max(boxStartY, previousBoxStartY);
|
||||
const xEndMin = Math.min(boxEndX, previousBoxEndX);
|
||||
const yEndMin = Math.min(boxEndY, previousBoxEndY);
|
||||
const intersection = (xEndMin - xStartMax) * (yEndMin - yStartMax);
|
||||
const boxArea = (boxEndX - boxStartX) * (boxEndY - boxStartY);
|
||||
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
|
||||
iou = intersection / (boxArea + previousBoxArea - intersection);
|
||||
}
|
||||
this.storedBoxes[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox;
|
||||
}
|
||||
}
|
||||
|
||||
exports.HandPipeline = HandPipeline;
|
||||
|
|
|
|||
|
|
@ -51,12 +51,12 @@ class HandPose {
|
|||
}
|
||||
}
|
||||
hands.push({
|
||||
confidence: prediction.handInViewConfidence,
|
||||
box: prediction.boundingBox ? [
|
||||
prediction.boundingBox.topLeft[0],
|
||||
prediction.boundingBox.topLeft[1],
|
||||
prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0],
|
||||
prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1],
|
||||
confidence: prediction.confidence,
|
||||
box: prediction.box ? [
|
||||
prediction.box.topLeft[0],
|
||||
prediction.box.topLeft[1],
|
||||
prediction.box.bottomRight[0] - prediction.box.topLeft[0],
|
||||
prediction.box.bottomRight[1] - prediction.box.topLeft[1],
|
||||
] : 0,
|
||||
landmarks: prediction.landmarks,
|
||||
annotations,
|
||||
|
|
|
|||
Loading…
Reference in New Issue