ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update hand model

pull/50/head
Vladimir Mandic 2020-11-08 09:56:02 -05:00
parent 1c0499d797
commit a61b19ac0c
29 changed files with 178666 additions and 33914 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.js
View File

@ -113,6 +113,7 @@ export default {
scoreThreshold: 0.8, // threshold for deciding when to remove boxes based on score in non-maximum suppression scoreThreshold: 0.8, // threshold for deciding when to remove boxes based on score in non-maximum suppression
enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace
maxHands: 1, // maximum number of hands detected in the input, should be set to the minimum number for performance maxHands: 1, // maximum number of hands detected in the input, should be set to the minimum number for performance
landmarks: true, // detect hand landmarks or just hand boundary box
detector: { detector: {
modelPath: '../models/handdetect.json', modelPath: '../models/handdetect.json',
}, },

33
demo/browser.js
View File

@ -27,6 +27,10 @@ const ui = {
maxFrames: 10, maxFrames: 10,
modelsPreload: true, modelsPreload: true,
modelsWarmup: true, modelsWarmup: true,
menuWidth: 0,
menuHeight: 0,
camera: {},
fps: [],
}; };
// global variables // global variables
@ -34,8 +38,6 @@ let menu;
let menuFX; let menuFX;
let worker; let worker;
let timeStamp; let timeStamp;
let camera = {};
const fps = [];
// helper function: translates json to human readable string // helper function: translates json to human readable string
function str(...msg) { function str(...msg) {
@ -62,17 +64,22 @@ const status = (msg) => {
// draws processed results and starts processing of a next frame // draws processed results and starts processing of a next frame
function drawResults(input, result, canvas) { function drawResults(input, result, canvas) {
// update fps data // update fps data
fps.push(1000 / (performance.now() - timeStamp)); const elapsed = performance.now() - timeStamp;
if (fps.length > ui.maxFrames) fps.shift(); ui.fps.push(1000 / elapsed);
if (ui.fps.length > ui.maxFrames) ui.fps.shift();
// enable for continous performance monitoring // enable for continous performance monitoring
// console.log(result.performance); // console.log(result.performance);
// immediate loop before we even draw results, but limit frame rate to 30
if (input.srcObject) {
// eslint-disable-next-line no-use-before-define // eslint-disable-next-line no-use-before-define
if (input.srcObject) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop before we even draw results if (elapsed > 33) requestAnimationFrame(() => runHumanDetect(input, canvas));
// eslint-disable-next-line no-use-before-define
else setTimeout(() => runHumanDetect(input, canvas), 33 - elapsed);
}
// draw fps chart // draw fps chart
menu.updateChart('FPS', fps); menu.updateChart('FPS', ui.fps);
// draw image from video // draw image from video
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.fillStyle = ui.baseBackground; ctx.fillStyle = ui.baseBackground;
@ -94,9 +101,9 @@ function drawResults(input, result, canvas) {
const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : ''; const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : '';
const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`; const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`;
const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : ''; const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : '';
const avg = Math.trunc(10 * fps.reduce((a, b) => a + b) / fps.length) / 10; const avg = Math.trunc(10 * ui.fps.reduce((a, b) => a + b) / ui.fps.length) / 10;
document.getElementById('log').innerText = ` document.getElementById('log').innerText = `
video: ${camera.name} | facing: ${camera.facing} | resolution: ${camera.width} x ${camera.height} ${processing} video: ${ui.camera.name} | facing: ${ui.camera.facing} | resolution: ${ui.camera.width} x ${ui.camera.height} ${processing}
backend: ${human.tf.getBackend()} | ${memory} backend: ${human.tf.getBackend()} | ${memory}
performance: ${str(result.performance)} FPS:${avg} performance: ${str(result.performance)} FPS:${avg}
`; `;
@ -147,7 +154,7 @@ async function setupCamera() {
const track = stream.getVideoTracks()[0]; const track = stream.getVideoTracks()[0];
const settings = track.getSettings(); const settings = track.getSettings();
log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track); log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' }; ui.camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
return new Promise((resolve) => { return new Promise((resolve) => {
video.onloadeddata = async () => { video.onloadeddata = async () => {
video.width = video.videoWidth; video.width = video.videoWidth;
@ -156,6 +163,8 @@ async function setupCamera() {
canvas.height = video.height; canvas.height = video.height;
canvas.style.width = canvas.width > canvas.height ? '100vw' : ''; canvas.style.width = canvas.width > canvas.height ? '100vw' : '';
canvas.style.height = canvas.width > canvas.height ? '' : '100vh'; canvas.style.height = canvas.width > canvas.height ? '' : '100vh';
ui.menuWidth.input.setAttribute('value', video.width);
ui.menuHeight.input.setAttribute('value', video.height);
// silly font resizing for paint-on-canvas since viewport can be zoomed // silly font resizing for paint-on-canvas since viewport can be zoomed
const size = 14 + (6 * canvas.width / window.innerWidth); const size = 14 + (6 * canvas.width / window.innerWidth);
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`); ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`);
@ -351,8 +360,8 @@ function setupMenu() {
menuFX.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">'); menuFX.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menuFX.addLabel('Image Processing'); menuFX.addLabel('Image Processing');
menuFX.addBool('Enabled', human.config.filter, 'enabled'); menuFX.addBool('Enabled', human.config.filter, 'enabled');
menuFX.addRange('Image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val)); ui.menuWidth = menuFX.addRange('Image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val));
menuFX.addRange('Image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val)); ui.menuHeight = menuFX.addRange('Image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val));
menuFX.addRange('Brightness', human.config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => human.config.filter.brightness = parseFloat(val)); menuFX.addRange('Brightness', human.config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => human.config.filter.brightness = parseFloat(val));
menuFX.addRange('Contrast', human.config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => human.config.filter.contrast = parseFloat(val)); menuFX.addRange('Contrast', human.config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => human.config.filter.contrast = parseFloat(val));
menuFX.addRange('Sharpness', human.config.filter, 'sharpness', 0, 1.0, 0.05, (val) => human.config.filter.sharpness = parseFloat(val)); menuFX.addRange('Sharpness', human.config.filter, 'sharpness', 0, 1.0, 0.05, (val) => human.config.filter.sharpness = parseFloat(val));

1
demo/menu.js
View File

@ -219,6 +219,7 @@ class Menu {
evt.target.setAttribute('value', evt.target.value); evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target.value); if (callback) callback(evt.target.value);
}); });
el.input = el.children[0];
return el; return el;
} }

4
changelog.js → dev-server/changelog.js
View File

@ -3,7 +3,7 @@ const path = require('path');
const dayjs = require('dayjs'); const dayjs = require('dayjs');
const simpleGit = require('simple-git/promise'); const simpleGit = require('simple-git/promise');
const logger = require('@vladmandic/pilogger'); const logger = require('@vladmandic/pilogger');
const app = require('./package.json'); const app = require('../package.json');
const git = simpleGit(); const git = simpleGit();
@ -45,5 +45,5 @@ async function update(f) {
exports.update = update; exports.update = update;
if (!module.parent) { if (!module.parent) {
update('wiki/Change-Log.md'); update('../wiki/Change-Log.md');
} }

0
dev-server.crt → dev-server/dev-server.crt
View File

6
dev-server.js → dev-server/dev-server.js
View File

@ -26,9 +26,9 @@ const log = require('@vladmandic/pilogger');
const options = { const options = {
// key: fs.readFileSync('/home/vlado/dev/piproxy/cert/private.pem'), // key: fs.readFileSync('/home/vlado/dev/piproxy/cert/private.pem'),
// cert: fs.readFileSync('/home/vlado/dev/piproxy/cert/fullchain.pem'), // cert: fs.readFileSync('/home/vlado/dev/piproxy/cert/fullchain.pem'),
key: fs.readFileSync('./dev-server.key'), key: fs.readFileSync('dev-server/dev-server.key'),
cert: fs.readFileSync('./dev-server.crt'), cert: fs.readFileSync('dev-server/dev-server.crt'),
root: '.', root: '..',
default: 'demo/index.html', default: 'demo/index.html',
port: 8000, port: 8000,
monitor: ['package.json', 'config.js', 'demo', 'src'], monitor: ['package.json', 'config.js', 'demo', 'src'],

0
dev-server.key → dev-server/dev-server.key
View File

117890
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

21
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"demo/browser.js": { "demo/browser.js": {
"bytes": 17856, "bytes": 18278,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js" "path": "dist/human.esm.js"
@ -19,11 +19,11 @@
"imports": [] "imports": []
}, },
"demo/menu.js": { "demo/menu.js": {
"bytes": 12676, "bytes": 12707,
"imports": [] "imports": []
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1278200, "bytes": 3196462,
"imports": [] "imports": []
} }
}, },
@ -31,28 +31,25 @@
"dist/demo-browser-index.js.map": { "dist/demo-browser-index.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5532275 "bytes": 5560779
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1664606 "bytesInOutput": 3194325
},
"dist/human.esm.js": {
"bytesInOutput": 8716
}, },
"demo/draw.js": { "demo/draw.js": {
"bytesInOutput": 7451 "bytesInOutput": 7453
}, },
"demo/menu.js": { "demo/menu.js": {
"bytesInOutput": 12678 "bytesInOutput": 12709
}, },
"demo/browser.js": { "demo/browser.js": {
"bytesInOutput": 15855 "bytesInOutput": 16220
} }
}, },
"bytes": 1709428 "bytes": 3230829
} }
} }
} }

2
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.json vendored
View File

@ -5,7 +5,7 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 3374, "bytes": 3417,
"imports": [] "imports": []
}, },
"src/age/age.js": { "src/age/age.js": {
@ -406,7 +406,7 @@
"bytesInOutput": 1325 "bytesInOutput": 1325
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3047
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 7201 "bytesInOutput": 7201
@ -415,7 +415,7 @@
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 216530 "bytes": 216573
} }
} }
} }

95133
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

124
dist/human.esm.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"config.js": { "config.js": {
"bytes": 7664, "bytes": 7744,
"imports": [] "imports": []
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": { "node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
@ -149,7 +149,7 @@
] ]
}, },
"package.json": { "package.json": {
"bytes": 3374, "bytes": 3417,
"imports": [] "imports": []
}, },
"src/age/age.js": { "src/age/age.js": {
@ -379,7 +379,7 @@
"imports": [] "imports": []
}, },
"src/hand/box.js": { "src/hand/box.js": {
"bytes": 3192, "bytes": 3220,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -387,7 +387,7 @@
] ]
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytes": 4220, "bytes": 4229,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -398,7 +398,7 @@
] ]
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytes": 8214, "bytes": 7445,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -412,7 +412,7 @@
] ]
}, },
"src/hand/handpose.js": { "src/hand/handpose.js": {
"bytes": 3095, "bytes": 3029,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,178 +513,178 @@
"dist/human.esm.js.map": { "dist/human.esm.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5418726 "bytes": 5609744
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
"inputs": { "inputs": {
"empty:/home/vlado/dev/human/node_modules/node-fetch/browser.js": { "empty:/home/vlado/dev/human/node_modules/node-fetch/browser.js": {
"bytesInOutput": 13 "bytesInOutput": 45
}, },
"empty:util": { "empty:util": {
"bytesInOutput": 13 "bytesInOutput": 42
}, },
"empty:crypto": { "empty:crypto": {
"bytesInOutput": 13 "bytesInOutput": 44
}, },
"node_modules/@tensorflow/tfjs-core/dist/tf-core.node.js": { "node_modules/@tensorflow/tfjs-core/dist/tf-core.node.js": {
"bytesInOutput": 295162 "bytesInOutput": 1010341
}, },
"node_modules/@tensorflow/tfjs-layers/dist/tf-layers.node.js": { "node_modules/@tensorflow/tfjs-layers/dist/tf-layers.node.js": {
"bytesInOutput": 238778 "bytesInOutput": 514491
}, },
"node_modules/@tensorflow/tfjs-converter/dist/tf-converter.node.js": { "node_modules/@tensorflow/tfjs-converter/dist/tf-converter.node.js": {
"bytesInOutput": 115231 "bytesInOutput": 258962
}, },
"empty:/home/vlado/dev/human/node_modules/string_decoder/lib/string_decoder.js": { "empty:/home/vlado/dev/human/node_modules/string_decoder/lib/string_decoder.js": {
"bytesInOutput": 13 "bytesInOutput": 52
}, },
"node_modules/@tensorflow/tfjs-data/dist/tf-data.node.js": { "node_modules/@tensorflow/tfjs-data/dist/tf-data.node.js": {
"bytesInOutput": 52364 "bytesInOutput": 129585
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/alea.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/alea.js": {
"bytesInOutput": 990 "bytesInOutput": 2112
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor128.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor128.js": {
"bytesInOutput": 755 "bytesInOutput": 1699
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorwow.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorwow.js": {
"bytesInOutput": 845 "bytesInOutput": 1897
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorshift7.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xorshift7.js": {
"bytesInOutput": 1001 "bytesInOutput": 2307
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor4096.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/xor4096.js": {
"bytesInOutput": 1164 "bytesInOutput": 2742
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/tychei.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/lib/tychei.js": {
"bytesInOutput": 880 "bytesInOutput": 1940
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/seedrandom.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/seedrandom.js": {
"bytesInOutput": 1614 "bytesInOutput": 4019
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/index.js": { "node_modules/@tensorflow/tfjs-backend-cpu/node_modules/seedrandom/index.js": {
"bytesInOutput": 171 "bytesInOutput": 458
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": { "node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
"bytesInOutput": 82512 "bytesInOutput": 272412
}, },
"node_modules/@tensorflow/tfjs-backend-webgl/dist/tf-backend-webgl.node.js": { "node_modules/@tensorflow/tfjs-backend-webgl/dist/tf-backend-webgl.node.js": {
"bytesInOutput": 261415 "bytesInOutput": 561667
}, },
"node_modules/@tensorflow/tfjs/dist/tf.node.js": { "node_modules/@tensorflow/tfjs/dist/tf.node.js": {
"bytesInOutput": 760 "bytesInOutput": 3025
}, },
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytesInOutput": 3091 "bytesInOutput": 7099
}, },
"src/face/keypoints.js": { "src/face/keypoints.js": {
"bytesInOutput": 1946 "bytesInOutput": 2768
}, },
"src/face/box.js": { "src/face/box.js": {
"bytesInOutput": 1006 "bytesInOutput": 2070
}, },
"src/face/util.js": { "src/face/util.js": {
"bytesInOutput": 1190 "bytesInOutput": 3017
}, },
"src/face/facepipeline.js": { "src/face/facepipeline.js": {
"bytesInOutput": 5656 "bytesInOutput": 13567
}, },
"src/face/uvcoords.js": { "src/face/uvcoords.js": {
"bytesInOutput": 16786 "bytesInOutput": 20584
}, },
"src/face/triangulation.js": { "src/face/triangulation.js": {
"bytesInOutput": 9991 "bytesInOutput": 23309
}, },
"src/face/facemesh.js": { "src/face/facemesh.js": {
"bytesInOutput": 1286 "bytesInOutput": 2590
}, },
"src/profile.js": { "src/profile.js": {
"bytesInOutput": 620 "bytesInOutput": 1092
}, },
"src/age/age.js": { "src/age/age.js": {
"bytesInOutput": 972 "bytesInOutput": 1843
}, },
"src/gender/gender.js": { "src/gender/gender.js": {
"bytesInOutput": 1471 "bytesInOutput": 2996
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1428 "bytesInOutput": 2715
}, },
"src/body/modelBase.js": { "src/body/modelBase.js": {
"bytesInOutput": 433 "bytesInOutput": 900
}, },
"src/body/modelMobileNet.js": { "src/body/modelMobileNet.js": {
"bytesInOutput": 245 "bytesInOutput": 494
}, },
"src/body/heapSort.js": { "src/body/heapSort.js": {
"bytesInOutput": 1042 "bytesInOutput": 1637
}, },
"src/body/buildParts.js": { "src/body/buildParts.js": {
"bytesInOutput": 547 "bytesInOutput": 1752
}, },
"src/body/keypoints.js": { "src/body/keypoints.js": {
"bytesInOutput": 1633 "bytesInOutput": 2277
}, },
"src/body/vectors.js": { "src/body/vectors.js": {
"bytesInOutput": 616 "bytesInOutput": 1408
}, },
"src/body/decodePose.js": { "src/body/decodePose.js": {
"bytesInOutput": 1024 "bytesInOutput": 3773
}, },
"src/body/decodeMultiple.js": { "src/body/decodeMultiple.js": {
"bytesInOutput": 604 "bytesInOutput": 1990
}, },
"src/body/util.js": { "src/body/util.js": {
"bytesInOutput": 1062 "bytesInOutput": 2398
}, },
"src/body/modelPoseNet.js": { "src/body/modelPoseNet.js": {
"bytesInOutput": 916 "bytesInOutput": 2100
}, },
"src/body/posenet.js": { "src/body/posenet.js": {
"bytesInOutput": 474 "bytesInOutput": 903
}, },
"src/hand/box.js": { "src/hand/box.js": {
"bytesInOutput": 1398 "bytesInOutput": 3583
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 1754 "bytesInOutput": 4485
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 1005 "bytesInOutput": 3419
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 2876 "bytesInOutput": 7278
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127001 "bytesInOutput": 256590
}, },
"src/hand/handpose.js": { "src/hand/handpose.js": {
"bytesInOutput": 1263 "bytesInOutput": 3058
}, },
"src/gesture.js": { "src/gesture.js": {
"bytesInOutput": 1220 "bytesInOutput": 2270
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11014 "bytesInOutput": 20097
}, },
"src/image.js": { "src/image.js": {
"bytesInOutput": 2365 "bytesInOutput": 4482
}, },
"config.js": { "config.js": {
"bytesInOutput": 1326 "bytesInOutput": 2299
}, },
"package.json": { "package.json": {
"bytesInOutput": 3005 "bytesInOutput": 3561
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 7219 "bytesInOutput": 11575
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 1278200 "bytes": 3196462
} }
} }
} }

2
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.json vendored
View File

@ -149,7 +149,7 @@
] ]
}, },
"package.json": { "package.json": {
"bytes": 3374, "bytes": 3417,
"imports": [] "imports": []
}, },
"src/age/age.js": { "src/age/age.js": {
@ -675,13 +675,13 @@
"bytesInOutput": 1326 "bytesInOutput": 1326
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3047
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 7257 "bytesInOutput": 7257
} }
}, },
"bytes": 1278245 "bytes": 1278288
} }
} }
} }

2
dist/human.node-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.node-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.json vendored
View File

@ -5,7 +5,7 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 3374, "bytes": 3417,
"imports": [] "imports": []
}, },
"src/age/age.js": { "src/age/age.js": {
@ -406,7 +406,7 @@
"bytesInOutput": 1324 "bytesInOutput": 1324
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3047
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 28 "bytesInOutput": 28
@ -415,7 +415,7 @@
"bytesInOutput": 7201 "bytesInOutput": 7201
} }
}, },
"bytes": 216537 "bytes": 216580
} }
} }
} }

8
package.json
View File

@ -41,16 +41,16 @@
"scripts": { "scripts": {
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js", "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",
"lint": "eslint src/*.js demo/*.js", "lint": "eslint src/*.js demo/*.js",
"dev": "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server.js", "dev": "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server/dev-server.js",
"changelog": "node dev-server/changelog.js",
"build-iife": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=iife --external:fs --global-name=Human --metafile=dist/human.json --outfile=dist/human.js src/human.js", "build-iife": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=iife --external:fs --global-name=Human --metafile=dist/human.json --outfile=dist/human.js src/human.js",
"build-esm-bundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js", "build-esm-bundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js",
"build-esm-nobundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js", "build-esm-nobundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js",
"build-node": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --metafile=dist/human.node.json --outfile=dist/human.node.js src/human.js", "build-node": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --metafile=dist/human.node.json --outfile=dist/human.node.js src/human.js",
"build-node-nobundle": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --external:@tensorflow --metafile=dist/human.node.json --outfile=dist/human.node-nobundle.js src/human.js", "build-node-nobundle": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --external:@tensorflow --metafile=dist/human.node.json --outfile=dist/human.node-nobundle.js src/human.js",
"build-demo": "esbuild --bundle --log-level=error --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/demo-browser-index.json --outfile=dist/demo-browser-index.js demo/browser.js", "build-demo": "esbuild --bundle --log-level=error --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/demo-browser-index.json --outfile=dist/demo-browser-index.js demo/browser.js",
"build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo", "build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo && npm run changelog",
"update": "npm update --depth 20 --force && npm dedupe && npm prune && npm audit", "update": "npm update --depth 20 --force && npm dedupe && npm prune && npm audit"
"changelog": "node changelog.js"
}, },
"keywords": [ "keywords": [
"tensorflowjs", "tensorflowjs",

2
src/hand/box.js
View File

@ -46,7 +46,7 @@ function scaleBoxCoordinates(box, factor) {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]]; const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord; return scaledCoord;
}); });
return { startPoint, endPoint, palmLandmarks }; return { startPoint, endPoint, palmLandmarks, confidence: box.confidence };
} }
function enlargeBox(box, factor = 1.5) { function enlargeBox(box, factor = 1.5) {
const center = getBoxCenter(box); const center = getBoxCenter(box);

35
src/hand/handdetector.js
View File

@ -49,29 +49,28 @@ class HandDetector {
async getBoxes(input, config) { async getBoxes(input, config) {
const batched = this.model.predict(input); const batched = this.model.predict(input);
const predictions = batched.squeeze(); const predictions = batched.squeeze();
batched.dispose();
const scores = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze()); const scores = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze());
// const scoresVal = scores.dataSync(); // scoresVal[boxIndex] is box confidence const scoresVal = scores.dataSync();
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold); rawBoxes.dispose();
const boxesWithHands = boxesWithHandsT.arraySync(); const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
const toDispose = [ const filtered = filteredT.arraySync();
batched, scores.dispose();
boxesWithHandsT, filteredT.dispose();
predictions,
boxes,
rawBoxes,
scores,
];
const hands = []; const hands = [];
for (const boxIndex of boxesWithHands) { for (const boxIndex of filtered) {
if (scoresVal[boxIndex] >= config.minConfidence) {
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]); const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]);
const rawPalmLandmarks = tf.slice(predictions, [boxIndex, 5], [1, 14]); const rawPalmLandmarks = tf.slice(predictions, [boxIndex, 5], [1, 14]);
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2])); const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2]));
rawPalmLandmarks.dispose(); rawPalmLandmarks.dispose();
hands.push({ box: matchingBox, palmLandmarks }); hands.push({ box: matchingBox, palmLandmarks, confidence: scoresVal[boxIndex] });
} }
toDispose.forEach((tensor) => tensor.dispose()); }
predictions.dispose();
boxes.dispose();
return hands; return hands;
} }
@ -84,13 +83,13 @@ class HandDetector {
if (!predictions || predictions.length === 0) return null; if (!predictions || predictions.length === 0) return null;
const hands = []; const hands = [];
for (const prediction of predictions) { for (const prediction of predictions) {
const boundingBoxes = prediction.box.dataSync(); const boxes = prediction.box.dataSync();
const startPoint = boundingBoxes.slice(0, 2); const startPoint = boxes.slice(0, 2);
const endPoint = boundingBoxes.slice(2, 4); const endPoint = boxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.box.dispose(); prediction.box.dispose();
prediction.palmLandmarks.dispose(); prediction.palmLandmarks.dispose();
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / config.inputSize, inputHeight / config.inputSize])); hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / config.inputSize, inputHeight / config.inputSize]));
} }
return hands; return hands;
} }

62
src/hand/handpipeline.js
View File

@ -19,7 +19,6 @@ const tf = require('@tensorflow/tfjs');
const box = require('./box'); const box = require('./box');
const util = require('./util'); const util = require('./util');
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8;
const PALM_BOX_SHIFT_VECTOR = [0, -0.4]; const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
const PALM_BOX_ENLARGE_FACTOR = 3; const PALM_BOX_ENLARGE_FACTOR = 3;
const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
@ -87,23 +86,29 @@ class HandPipeline {
async estimateHands(image, config) { async estimateHands(image, config) {
this.skipped++; this.skipped++;
let useFreshBox = false; let useFreshBox = false;
// run new detector every skipFrames
const boxes = (this.skipped > config.skipFrames) // run new detector every skipFrames unless we only want box to start with
? await this.boxDetector.estimateHandBounds(image, config) : null; let boxes;
if ((this.skipped > config.skipFrames) || !config.landmarks) {
boxes = await this.boxDetector.estimateHandBounds(image, config);
this.skipped = 0;
}
// if detector result count doesn't match current working set, use it to reset current working set // if detector result count doesn't match current working set, use it to reset current working set
if (boxes && (boxes.length !== this.detectedHands) && (this.detectedHands !== config.maxHands)) { if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.maxHands) || !config.landmarks)) {
// console.log(this.skipped, config.maxHands, this.detectedHands, this.storedBoxes.length, boxes.length);
this.storedBoxes = []; this.storedBoxes = [];
this.detectedHands = 0; this.detectedHands = 0;
for (const possible of boxes) this.storedBoxes.push(possible); for (const possible of boxes) this.storedBoxes.push(possible);
if (this.storedBoxes.length > 0) useFreshBox = true; if (this.storedBoxes.length > 0) useFreshBox = true;
this.skipped = 0;
} }
const hands = []; const hands = [];
// console.log(`skipped: ${this.skipped} max: ${config.maxHands} detected: ${this.detectedHands} stored: ${this.storedBoxes.length} new: ${boxes?.length}`);
// go through working set of boxes // go through working set of boxes
for (const i in this.storedBoxes) { for (const i in this.storedBoxes) {
const currentBox = this.storedBoxes[i]; const currentBox = this.storedBoxes[i];
if (!currentBox) continue; if (!currentBox) continue;
if (config.landmarks) {
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]); const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
const palmCenter = box.getBoxCenter(currentBox); const palmCenter = box.getBoxCenter(currentBox);
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
@ -125,30 +130,31 @@ class HandPipeline {
keypointsReshaped.dispose(); keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords); const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateStoredBoxes(nextBoundingBox, i); this.storedBoxes[i] = nextBoundingBox;
const result = { const result = {
landmarks: coords, landmarks: coords,
handInViewConfidence: confidenceValue, confidence: confidenceValue,
boundingBox: { box: {
topLeft: nextBoundingBox.startPoint, topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint, bottomRight: nextBoundingBox.endPoint,
}, },
}; };
hands.push(result); hands.push(result);
} else { } else {
this.updateStoredBoxes(null, i); this.storedBoxes[i] = null;
/* }
keypoints.dispose();
} else {
const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
const result = { const result = {
handInViewConfidence: confidenceValue, confidence: currentBox.confidence,
boundingBox: { box: {
topLeft: currentBox.startPoint, topLeft: enlarged.startPoint,
bottomRight: currentBox.endPoint, bottomRight: enlarged.endPoint,
}, },
}; };
hands.push(result); hands.push(result);
*/
} }
keypoints.dispose();
} }
this.storedBoxes = this.storedBoxes.filter((a) => a !== null); this.storedBoxes = this.storedBoxes.filter((a) => a !== null);
this.detectedHands = hands.length; this.detectedHands = hands.length;
@ -163,26 +169,6 @@ class HandPipeline {
const endPoint = [Math.max(...xs), Math.max(...ys)]; const endPoint = [Math.max(...xs), Math.max(...ys)];
return { startPoint, endPoint }; return { startPoint, endPoint };
} }
updateStoredBoxes(newBox, i) {
const previousBox = this.storedBoxes[i];
let iou = 0;
if (newBox && previousBox && previousBox.startPoint) {
const [boxStartX, boxStartY] = newBox.startPoint;
const [boxEndX, boxEndY] = newBox.endPoint;
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;
const [previousBoxEndX, previousBoxEndY] = previousBox.endPoint;
const xStartMax = Math.max(boxStartX, previousBoxStartX);
const yStartMax = Math.max(boxStartY, previousBoxStartY);
const xEndMin = Math.min(boxEndX, previousBoxEndX);
const yEndMin = Math.min(boxEndY, previousBoxEndY);
const intersection = (xEndMin - xStartMax) * (yEndMin - yStartMax);
const boxArea = (boxEndX - boxStartX) * (boxEndY - boxStartY);
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
iou = intersection / (boxArea + previousBoxArea - intersection);
}
this.storedBoxes[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox;
}
} }
exports.HandPipeline = HandPipeline; exports.HandPipeline = HandPipeline;

12
src/hand/handpose.js
View File

@ -51,12 +51,12 @@ class HandPose {
} }
} }
hands.push({ hands.push({
confidence: prediction.handInViewConfidence, confidence: prediction.confidence,
box: prediction.boundingBox ? [ box: prediction.box ? [
prediction.boundingBox.topLeft[0], prediction.box.topLeft[0],
prediction.boundingBox.topLeft[1], prediction.box.topLeft[1],
prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0], prediction.box.bottomRight[0] - prediction.box.topLeft[0],
prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1], prediction.box.bottomRight[1] - prediction.box.topLeft[1],
] : 0, ] : 0,
landmarks: prediction.landmarks, landmarks: prediction.landmarks,
annotations, annotations,