ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add curve draw output

pull/91/head
Vladimir Mandic 2021-03-06 10:38:04 -05:00
parent eac219509e
commit 456d34ee22
34 changed files with 1164 additions and 1247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.js
View File

@ -71,7 +71,7 @@ export default {
// 'blazeface-back' is blazeface model optimized for smaller and/or distanct faces // 'blazeface-back' is blazeface model optimized for smaller and/or distanct faces
// 'faceboxes' is alternative model to 'blazeface' // 'faceboxes' is alternative model to 'blazeface'
inputSize: 256, // fixed value: 128 for front and 256 for 'back' inputSize: 256, // fixed value: 128 for front and 256 for 'back'
rotation: false, // use best-guess rotated face image or just box with rotation as-is rotation: true, // use best-guess rotated face image or just box with rotation as-is
// false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees // false means higher performance, but incorrect mesh mapping if face angle is above 20 degrees
maxFaces: 10, // maximum number of faces detected in the input maxFaces: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance

23
demo/browser.js
View File

@ -2,7 +2,7 @@ import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
import Menu from './menu.js'; import Menu from './menu.js';
import GLBench from './gl-bench.js'; import GLBench from './gl-bench.js';
const userConfig = { }; // add any user configuration overrides const userConfig = { backend: 'wasm' }; // add any user configuration overrides
/* /*
const userConfig = { const userConfig = {
@ -40,11 +40,11 @@ const ui = {
drawFPS: [], // internal, holds fps values for draw performance drawFPS: [], // internal, holds fps values for draw performance
buffered: false, // experimental, should output be buffered between frames buffered: false, // experimental, should output be buffered between frames
drawWarmup: false, // debug only, should warmup image processing be displayed on startup drawWarmup: false, // debug only, should warmup image processing be displayed on startup
drawThread: null, // perform draw operations in a separate thread drawThread: null, // internl, perform draw operations in a separate thread
detectThread: null, // perform detect operations in a separate thread detectThread: null, // internl, perform detect operations in a separate thread
framesDraw: 0, // internal, statistics on frames drawn framesDraw: 0, // internal, statistics on frames drawn
framesDetect: 0, // internal, statistics on frames detected framesDetect: 0, // internal, statistics on frames detected
bench: false, // show gl fps benchmark window bench: true, // show gl fps benchmark window
lastFrame: 0, // time of last frame processing lastFrame: 0, // time of last frame processing
}; };
@ -110,7 +110,11 @@ async function drawResults(input) {
await menu.process.updateChart('FPS', ui.detectFPS); await menu.process.updateChart('FPS', ui.detectFPS);
// get updated canvas // get updated canvas
if (ui.buffered || !result.canvas) result.canvas = await human.image(input).canvas; if (ui.buffered || !result.canvas) {
const image = await human.image(input);
result.canvas = image.canvas;
human.tf.dispose(image.tensor);
}
// draw image from video // draw image from video
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
@ -421,10 +425,10 @@ function setupMenu() {
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth'); menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('print labels', human.draw.options, 'drawLabels'); menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes'); menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons'); menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons');
menu.display.addBool('Fill Polygons', human.draw.options, 'fillPolygons'); menu.display.addBool('fill polygons', human.draw.options, 'fillPolygons');
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] }); menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] });
menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val); menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val);
@ -449,8 +453,8 @@ function setupMenu() {
menu.process = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[2] }); menu.process = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[2] });
menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val); menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val);
menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val); menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val);
menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val); // menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val);
menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val); // menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val);
menu.process.addBool('use web worker', ui, 'useWorker'); menu.process.addBool('use web worker', ui, 'useWorker');
menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.process.addLabel('model parameters'); menu.process.addLabel('model parameters');
@ -526,7 +530,6 @@ async function drawWarmup(res) {
async function main() { async function main() {
log('Demo starting ...'); log('Demo starting ...');
log('Browser:', navigator?.userAgent);
setupMenu(); setupMenu();
document.getElementById('log').innerText = `Human: version ${human.version}`; document.getElementById('log').innerText = `Human: version ${human.version}`;
if (ui.modelsPreload && !ui.useWorker) { if (ui.modelsPreload && !ui.useWorker) {

2
demo/gl-bench.js
View File

@ -20,7 +20,7 @@ const UISVG = `
<svg viewBox="0 0 55 60"> <svg viewBox="0 0 55 60">
<text x="27" y="56" class="gl-fps">00 FPS</text> <text x="27" y="56" class="gl-fps">00 FPS</text>
<text x="30" y="8" class="gl-mem"></text> <text x="30" y="8" class="gl-mem"></text>
<rect x="0" y="14" rx="4" ry="4" width="55" height="32"></rect> <rect x="0" y="14" rx="4" ry="4" width="65" height="32"></rect>
<polyline class="gl-chart"></polyline> <polyline class="gl-chart"></polyline>
</svg> </svg>
<svg viewBox="0 0 14 60" class="gl-cpu-svg"> <svg viewBox="0 0 14 60" class="gl-cpu-svg">

2
demo/menu.js
View File

@ -18,7 +18,7 @@ let theme = {
function createCSS() { function createCSS() {
if (CSScreated) return; if (CSScreated) return;
const css = ` const css = `
:root { --rounded: 0.2rem; } :root { --rounded: 0.1rem; }
.menu { position: absolute; top: 0rem; right: 0; width: max-content; padding: 0 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; .menu { position: absolute; top: 0rem; right: 0; width: max-content; padding: 0 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10;
box-shadow: 0 0 8px dimgrey; background: ${theme.background}; border-radius: var(--rounded); border-color: black; border-style: solid; border-width: thin; } box-shadow: 0 0 8px dimgrey; background: ${theme.background}; border-radius: var(--rounded); border-color: black; border-style: solid; border-width: thin; }

26
demo/node.js
View File

@ -40,8 +40,6 @@ async function init() {
// pre-load models // pre-load models
log.info('Human:', human.version); log.info('Human:', human.version);
log.info('Active Configuration', human.config); log.info('Active Configuration', human.config);
log.info('TFJS Version:', human.tf.version_core, 'Backend:', tf.getBackend());
log.info('TFJS Flags:', human.tf.env().features);
await human.load(); await human.load();
const loaded = Object.keys(human.models).filter((a) => human.models[a]); const loaded = Object.keys(human.models).filter((a) => human.models[a]);
log.info('Loaded:', loaded); log.info('Loaded:', loaded);
@ -63,27 +61,31 @@ async function detect(input) {
// dispose image tensor as we no longer need it // dispose image tensor as we no longer need it
image.dispose(); image.dispose();
// print data to console // print data to console
log.data(result); log.data('Face: ', result.face);
log.data('Body:', result.body);
log.data('Hand:', result.hand);
log.data('Gesture:', result.gesture);
} }
async function test() { async function test() {
// test with embedded face image // test with embedded full body image
let result;
log.state('Processing embedded warmup image: face'); log.state('Processing embedded warmup image: face');
myConfig.warmup = 'face'; myConfig.warmup = 'face';
const resultFace = await human.warmup(myConfig); result = await human.warmup(myConfig);
log.data('Face: ', resultFace.face); log.data('Face: ', result.face);
// test with embedded full body image
log.state('Processing embedded warmup image: full'); log.state('Processing embedded warmup image: full');
myConfig.warmup = 'full'; myConfig.warmup = 'full';
const resultFull = await human.warmup(myConfig); result = await human.warmup(myConfig);
log.data('Body:', resultFull.body); log.data('Body:', result.body);
log.data('Hand:', resultFull.hand); log.data('Hand:', result.hand);
log.data('Gesture:', resultFull.gesture); log.data('Gesture:', result.gesture);
} }
async function main() { async function main() {
log.info('NodeJS:', process.version); log.header();
log.info('Current folder:', process.env.PWD); log.info('Current folder:', process.env.PWD);
await init(); await init();
if (process.argv.length !== 3) { if (process.argv.length !== 3) {

578
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

12
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1351634, "bytes": 1352771,
"imports": [] "imports": []
}, },
"demo/menu.js": { "demo/menu.js": {
@ -13,7 +13,7 @@
"imports": [] "imports": []
}, },
"demo/browser.js": { "demo/browser.js": {
"bytes": 27915, "bytes": 27984,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js", "path": "dist/human.esm.js",
@ -35,7 +35,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 2061107 "bytes": 2059743
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
@ -43,7 +43,7 @@
"entryPoint": "demo/browser.js", "entryPoint": "demo/browser.js",
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1344140 "bytesInOutput": 1345275
}, },
"demo/menu.js": { "demo/menu.js": {
"bytesInOutput": 10696 "bytesInOutput": 10696
@ -52,10 +52,10 @@
"bytesInOutput": 6759 "bytesInOutput": 6759
}, },
"demo/browser.js": { "demo/browser.js": {
"bytesInOutput": 17673 "bytesInOutput": 17494
} }
}, },
"bytes": 1386653 "bytes": 1387609
} }
} }
} }

14
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

576
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

39
dist/human.esm.json vendored
View File

@ -4,6 +4,10 @@
"bytes": 401, "bytes": 401,
"imports": [] "imports": []
}, },
"src/sysinfo.ts": {
"bytes": 612,
"imports": []
},
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1065682, "bytes": 1065682,
"imports": [] "imports": []
@ -52,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14154, "bytes": 14094,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -274,7 +278,7 @@
] ]
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytes": 2017, "bytes": 2041,
"imports": [ "imports": [
{ {
"path": "src/posenet/keypoints.ts", "path": "src/posenet/keypoints.ts",
@ -388,7 +392,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 3259, "bytes": 2939,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -442,11 +446,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2594, "bytes": 2581,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
"bytes": 19435, "bytes": 16222,
"imports": [ "imports": [
{ {
"path": "config.js", "path": "config.js",
@ -459,12 +463,16 @@
] ]
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 21110, "bytes": 21589,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
"kind": "import-statement" "kind": "import-statement"
}, },
{
"path": "src/sysinfo.ts",
"kind": "import-statement"
},
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
"kind": "import-statement" "kind": "import-statement"
@ -545,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1979344 "bytes": 1977987
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -563,8 +571,11 @@
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"src/sysinfo.ts": {
"bytesInOutput": 394
},
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056715 "bytesInOutput": 1056721
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -582,10 +593,10 @@
"bytesInOutput": 28983 "bytesInOutput": 28983
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5054 "bytesInOutput": 5043
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11325 "bytesInOutput": 11629
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576 "bytesInOutput": 1576
@ -630,7 +641,7 @@
"bytesInOutput": 529 "bytesInOutput": 529
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytesInOutput": 354 "bytesInOutput": 378
}, },
"src/handpose/handpose.ts": { "src/handpose/handpose.ts": {
"bytesInOutput": 1281 "bytesInOutput": 1281
@ -672,13 +683,13 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2596 "bytesInOutput": 2583
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9164 "bytesInOutput": 9597
} }
}, },
"bytes": 1351634 "bytes": 1352771
} }
} }
} }

39
dist/human.iife.json vendored
View File

@ -4,6 +4,10 @@
"bytes": 401, "bytes": 401,
"imports": [] "imports": []
}, },
"src/sysinfo.ts": {
"bytes": 612,
"imports": []
},
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1065682, "bytes": 1065682,
"imports": [] "imports": []
@ -52,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14154, "bytes": 14094,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -274,7 +278,7 @@
] ]
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytes": 2017, "bytes": 2041,
"imports": [ "imports": [
{ {
"path": "src/posenet/keypoints.ts", "path": "src/posenet/keypoints.ts",
@ -388,7 +392,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 3259, "bytes": 2939,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -442,11 +446,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2594, "bytes": 2581,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
"bytes": 19435, "bytes": 16222,
"imports": [ "imports": [
{ {
"path": "config.js", "path": "config.js",
@ -459,12 +463,16 @@
] ]
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 21110, "bytes": 21589,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
"kind": "import-statement" "kind": "import-statement"
}, },
{
"path": "src/sysinfo.ts",
"kind": "import-statement"
},
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
"kind": "import-statement" "kind": "import-statement"
@ -545,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1979355 "bytes": 1977998
}, },
"dist/human.ts": { "dist/human.ts": {
"imports": [], "imports": [],
@ -559,13 +567,16 @@
"bytesInOutput": 1690 "bytesInOutput": 1690
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11361 "bytesInOutput": 11665
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"src/sysinfo.ts": {
"bytesInOutput": 394
},
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056715 "bytesInOutput": 1056721
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -583,7 +594,7 @@
"bytesInOutput": 28983 "bytesInOutput": 28983
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5054 "bytesInOutput": 5043
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576 "bytesInOutput": 1576
@ -628,7 +639,7 @@
"bytesInOutput": 529 "bytesInOutput": 529
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytesInOutput": 354 "bytesInOutput": 378
}, },
"src/handpose/handpose.ts": { "src/handpose/handpose.ts": {
"bytesInOutput": 1281 "bytesInOutput": 1281
@ -670,13 +681,13 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2596 "bytesInOutput": 2583
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9164 "bytesInOutput": 9597
} }
}, },
"bytes": 1351676 "bytes": 1352813
} }
} }
} }

16
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

16
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

37
dist/human.node.json vendored
View File

@ -4,6 +4,10 @@
"bytes": 401, "bytes": 401,
"imports": [] "imports": []
}, },
"src/sysinfo.ts": {
"bytes": 612,
"imports": []
},
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 737, "bytes": 737,
"imports": [] "imports": []
@ -52,7 +56,7 @@
"imports": [] "imports": []
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytes": 14154, "bytes": 14094,
"imports": [ "imports": [
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
@ -274,7 +278,7 @@
] ]
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytes": 2017, "bytes": 2041,
"imports": [ "imports": [
{ {
"path": "src/posenet/keypoints.ts", "path": "src/posenet/keypoints.ts",
@ -388,7 +392,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 3259, "bytes": 2939,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -442,11 +446,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2594, "bytes": 2581,
"imports": [] "imports": []
}, },
"src/draw.ts": { "src/draw.ts": {
"bytes": 19435, "bytes": 16222,
"imports": [ "imports": [
{ {
"path": "config.js", "path": "config.js",
@ -459,12 +463,16 @@
] ]
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 21110, "bytes": 21589,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
"kind": "import-statement" "kind": "import-statement"
}, },
{
"path": "src/sysinfo.ts",
"kind": "import-statement"
},
{ {
"path": "dist/tfjs.esm.js", "path": "dist/tfjs.esm.js",
"kind": "import-statement" "kind": "import-statement"
@ -545,7 +553,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 746101 "bytes": 744738
}, },
"dist/human.node-gpu.js": { "dist/human.node-gpu.js": {
"imports": [], "imports": [],
@ -562,11 +570,14 @@
"bytesInOutput": 1677 "bytesInOutput": 1677
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11332 "bytesInOutput": 11632
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 251 "bytesInOutput": 251
}, },
"src/sysinfo.ts": {
"bytesInOutput": 394
},
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1145 "bytesInOutput": 1145
}, },
@ -574,7 +585,7 @@
"bytesInOutput": 2338 "bytesInOutput": 2338
}, },
"src/blazeface/facepipeline.ts": { "src/blazeface/facepipeline.ts": {
"bytesInOutput": 5101 "bytesInOutput": 5090
}, },
"src/blazeface/box.ts": { "src/blazeface/box.ts": {
"bytesInOutput": 854 "bytesInOutput": 854
@ -628,7 +639,7 @@
"bytesInOutput": 525 "bytesInOutput": 525
}, },
"src/posenet/util.ts": { "src/posenet/util.ts": {
"bytesInOutput": 352 "bytesInOutput": 376
}, },
"src/handpose/handpose.ts": { "src/handpose/handpose.ts": {
"bytesInOutput": 1322 "bytesInOutput": 1322
@ -670,13 +681,13 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2593 "bytesInOutput": 2580
}, },
"src/draw.ts": { "src/draw.ts": {
"bytesInOutput": 9054 "bytesInOutput": 9486
} }
}, },
"bytes": 289059 "bytes": 290185
} }
} }
} }

576
dist/human.ts vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.ts.map vendored
View File

File diff suppressed because one or more lines are too long

12
package-lock.json generated
View File

@ -2866,9 +2866,9 @@
"dev": true "dev": true
}, },
"typescript": { "typescript": {
"version": "4.3.0-dev.20210305", "version": "4.2.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.0-dev.20210305.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.3.tgz",
"integrity": "sha512-OTALeeen7kl6FU1tcXRk3h+WY1NnE5lwyTGAZUCt9hw6tdaifgLXqEkfw9NHJc0xKV6PnU8GgnYFFVVyHLPSHg==", "integrity": "sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw==",
"dev": true "dev": true
}, },
"unbox-primitive": { "unbox-primitive": {
@ -2899,9 +2899,9 @@
"dev": true "dev": true
}, },
"v8-compile-cache": { "v8-compile-cache": {
"version": "2.2.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
"integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==", "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
"dev": true "dev": true
}, },
"validate-npm-package-license": { "validate-npm-package-license": {

2
package.json
View File

@ -50,7 +50,7 @@
"seedrandom": "^3.0.5", "seedrandom": "^3.0.5",
"simple-git": "^2.36.0", "simple-git": "^2.36.0",
"tslib": "^2.1.0", "tslib": "^2.1.0",
"typescript": "^4.3.0-dev.20210305" "typescript": "^4.2.3"
}, },
"scripts": { "scripts": {
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js", "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",

20
src/blazeface/facepipeline.ts
View File

@ -19,8 +19,9 @@ const IRIS_LOWER_CENTER_INDEX = 4;
const IRIS_IRIS_INDEX = 71; const IRIS_IRIS_INDEX = 71;
const IRIS_NUM_COORDINATES = 76; const IRIS_NUM_COORDINATES = 76;
// Replace the raw coordinates returned by facemesh with refined iris model coordinates. Update the z coordinate to be an average of the original and the new. This produces the best visual effect. // Replace the raw coordinates returned by facemesh with refined iris model coordinates
function replaceRawCoordinates(rawCoords, newCoords, prefix, keys = null) { // Update the z coordinate to be an average of the original and the new.
function replaceRawCoordinates(rawCoords, newCoords, prefix, keys) {
for (let i = 0; i < coords.MESH_TO_IRIS_INDICES_MAP.length; i++) { for (let i = 0; i < coords.MESH_TO_IRIS_INDICES_MAP.length; i++) {
const { key, indices } = coords.MESH_TO_IRIS_INDICES_MAP[i]; const { key, indices } = coords.MESH_TO_IRIS_INDICES_MAP[i];
const originalIndices = coords.MESH_ANNOTATIONS[`${prefix}${key}`]; const originalIndices = coords.MESH_ANNOTATIONS[`${prefix}${key}`];
@ -179,7 +180,7 @@ export class Pipeline {
}); });
} }
// log('face', `skipped: ${this.skipped} max: ${config.face.detector.maxFaces} detected: ${this.detectedFaces} stored: ${this.storedBoxes.length} new: ${detector?.boxes?.length}`); // console.log('face', `skipped: ${this.skipped} max: ${config.face.detector.maxFaces} detected: ${this.detectedFaces} stored: ${this.storedBoxes.length} new: ${detector?.boxes?.length}`);
let results = tf.tidy(() => this.storedBoxes.map((box, i) => { let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box). // The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
let face; let face;
@ -211,7 +212,7 @@ export class Pipeline {
return prediction; return prediction;
} }
const [, confidence, contourCoords] = this.meshDetector.predict(face); // The first returned tensor represents facial contours, which are included in the coordinates. const [, confidence, contourCoords] = this.meshDetector.predict(face); // The first returned tensor represents facial contours which are already included in the coordinates.
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = confidence.dataSync()[0];
if (faceConfidence < config.face.detector.minConfidence) return null; // if below confidence just exit if (faceConfidence < config.face.detector.minConfidence) return null; // if below confidence just exit
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]); const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
@ -228,14 +229,13 @@ export class Pipeline {
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize); const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords); const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) { // User is looking straight ahead. if (Math.abs(leftToRightEyeDepthDifference) < 30) { // User is looking straight ahead.
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left'); replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right'); replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', null);
// If the user is looking to the left or to the right, the iris coordinates tend to diverge too much from the mesh coordinates for them to be merged. So we only update a single contour line above and below the eye. // If the user is looking to the left or to the right, the iris coordinates tend to diverge too much from the mesh coordinates for them to be merged
// So we only update a single contour line above and below the eye.
} else if (leftToRightEyeDepthDifference < 1) { // User is looking towards the right. } else if (leftToRightEyeDepthDifference < 1) { // User is looking towards the right.
// @ts-ignore
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', ['EyeUpper0', 'EyeLower0']); replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', ['EyeUpper0', 'EyeLower0']);
} else { // User is looking towards the left. } else { // User is looking towards the left.
// @ts-ignore
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', ['EyeUpper0', 'EyeLower0']); replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', ['EyeUpper0', 'EyeLower0']);
} }
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, 'left'); const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, 'left');
@ -256,7 +256,7 @@ export class Pipeline {
rawCoords, rawCoords,
}; };
if (!config.face.mesh.returnRawData) delete prediction.rawCoords; if (!config.face.mesh.returnRawData) delete prediction.rawCoords;
this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoords.arraySync(), confidence: box.confidence, faceConfidence }; this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence };
return prediction; return prediction;
})); }));

13
src/blazepose/blazepose.ts
View File

@ -26,10 +26,11 @@ export async function predict(image, config) {
let points; let points;
if (!config.profile) { // run through profiler or just execute if (!config.profile) { // run through profiler or just execute
const resT = await model.predict(normalize); const resT = await model.predict(normalize);
// const segmentationT = resT.find((t) => (t.size === 16384 || t.size === 0)).squeeze(); // const segmentationT = resT.find((t) => (t.size === 16384))?.squeeze();
// const segmentation = segmentationT.arraySync(); // array 128 x 128 // const segmentation = segmentationT.arraySync(); // array 128 x 128
// tf.dispose(segmentationT); // segmentationT.dispose();
points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync(); // order of output tensors may change between models, full has 195 and upper has 155 items points = resT.find((t) => (t.size === 195 || t.size === 155)).dataSync(); // order of output tensors may change between models, full has 195 and upper has 155 items
// console.log(resT, points, segmentation);
resT.forEach((t) => t.dispose()); resT.forEach((t) => t.dispose());
} else { } else {
const profileData = await tf.profile(() => model.predict(normalize)); const profileData = await tf.profile(() => model.predict(normalize));
@ -57,11 +58,3 @@ export async function predict(image, config) {
// console.log('POINTS', imgSize, pts.length, pts); // console.log('POINTS', imgSize, pts.length, pts);
return [{ keypoints }]; return [{ keypoints }];
} }
/*
Model card:
- https://drive.google.com/file/d/10IU-DRP2ioSNjKFdiGbmmQX81xAYj88s/view
Download:
- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/058_BlazePose_Full_Keypoints/10_new_256x256/saved_model/tfjs_model_float16
- https://github.com/PINTO0309/PINTO_model_zoo/tree/main/053_BlazePose/20_new_256x256/saved_model/tfjs_model_float16
*/

265
src/draw.ts
View File

@ -9,13 +9,14 @@ export const options = {
lineHeight: 20, lineHeight: 20,
lineWidth: 6, lineWidth: 6,
pointSize: 2, pointSize: 2,
roundRect: 8, roundRect: 28,
drawPoints: false, drawPoints: false,
drawLabels: true, drawLabels: true,
drawBoxes: true, drawBoxes: true,
drawPolygons: true, drawPolygons: true,
fillPolygons: false, fillPolygons: false,
useDepth: true, useDepth: true,
useCurves: true,
bufferedOutput: false, bufferedOutput: false,
}; };
@ -27,9 +28,13 @@ function point(ctx, x, y) {
} }
function rect(ctx, x, y, width, height) { function rect(ctx, x, y, width, height) {
if (options.roundRect && options.roundRect > 0) { ctx.beginPath();
if (options.useCurves) {
const cx = (x + x + width) / 2;
const cy = (y + y + height) / 2;
ctx.ellipse(cx, cy, width / 2, height / 2, 0, 0, 2 * Math.PI);
} else {
ctx.lineWidth = options.lineWidth; ctx.lineWidth = options.lineWidth;
ctx.beginPath();
ctx.moveTo(x + options.roundRect, y); ctx.moveTo(x + options.roundRect, y);
ctx.lineTo(x + width - options.roundRect, y); ctx.lineTo(x + width - options.roundRect, y);
ctx.quadraticCurveTo(x + width, y, x + width, y + options.roundRect); ctx.quadraticCurveTo(x + width, y, x + width, y + options.roundRect);
@ -40,122 +45,40 @@ function rect(ctx, x, y, width, height) {
ctx.lineTo(x, y + options.roundRect); ctx.lineTo(x, y + options.roundRect);
ctx.quadraticCurveTo(x, y, x + options.roundRect, y); ctx.quadraticCurveTo(x, y, x + options.roundRect, y);
ctx.closePath(); ctx.closePath();
ctx.stroke();
} else {
rect(ctx, x, y, width, height);
} }
ctx.stroke();
} }
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function lines(ctx, points: number[] = []) {
function lines(ctx, points) { if (points === undefined || points.length === 0) return;
ctx.beginPath(); ctx.beginPath();
const path = new Path2D(); ctx.moveTo(points[0][0], points[0][1]);
path.moveTo(points[0][0], points[0][1]); for (const pt of points) ctx.lineTo(pt[0], parseInt(pt[1]));
for (const pt of points) { ctx.stroke();
path.lineTo(pt[0], parseInt(pt[1]));
}
ctx.stroke(path);
if (options.fillPolygons) { if (options.fillPolygons) {
ctx.closePath(); ctx.closePath();
ctx.fill(path); ctx.fill();
} }
} }
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function curves(ctx, points: number[] = []) {
function curve(ctx, points = []) { if (points === undefined || points.length === 0) return;
if (points.length < 2) return; if (!options.useCurves || points.length <= 2) {
ctx.lineWidth = options.lineWidth; lines(ctx, points);
ctx.beginPath(); return;
}
ctx.moveTo(points[0][0], points[0][1]); ctx.moveTo(points[0][0], points[0][1]);
for (let i = 0; i < points.length - 1; i++) { for (let i = 0; i < points.length - 2; i++) {
const xMid = (points[i][0] + points[i + 1][0]) / 2; const xc = (points[i][0] + points[i + 1][0]) / 2;
const yMid = (points[i][1] + points[i + 1][1]) / 2; const yc = (points[i][1] + points[i + 1][1]) / 2;
const cpX1 = (xMid + points[i][0]) / 2; ctx.quadraticCurveTo(points[i][0], points[i][1], xc, yc);
const cpX2 = (xMid + points[i + 1][1]) / 2;
ctx.quadraticCurveTo(cpX1, points[i][1], xMid, yMid);
ctx.quadraticCurveTo(cpX2, points[i + 1][1], points[i + 1][0], points[i + 1][0]);
} }
ctx.strokeStyle = options.color; ctx.quadraticCurveTo(points[points.length - 2][0], points[points.length - 2][1], points[points.length - 1][0], points[points.length - 1][1]);
ctx.stroke(); ctx.stroke();
} if (options.fillPolygons) {
ctx.closePath();
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars ctx.fill();
function bezier(ctx, points) {
const tension = 0; // tension at 0 will be straight line
const factor = 1; // factor is normally 1, but changing the value can control the smoothness too
if (points.length < 2) return;
ctx.lineWidth = options.lineWidth;
ctx.strokeStyle = options.color;
ctx.fillStyle = options.color;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
let dx1 = 0;
let dy1 = 0;
let preP = points[0];
for (let i = 1; i < points.length; i++) {
const curP = points[i];
const nexP = points[i + 1];
const m = nexP ? (nexP[1] - preP[1]) / (nexP[0] - preP[0]) : 0;
const dx2 = nexP ? (nexP[0] - curP[0]) * -factor : 0;
const dy2 = nexP ? dx2 * m * tension : 0;
ctx.bezierCurveTo(preP[0] - dx1, preP[1] - dy1, curP[0] + dx2, curP[1] + dy2, curP[0], curP[1]);
dx1 = dx2;
dy1 = dy2;
preP = curP;
} }
ctx.stroke();
}
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
function spline(ctx, points) {
const tension = 0.8;
if (points.length < 2) return;
const va = (arr, i, j) => [arr[2 * j] - arr[2 * i], arr[2 * j + 1] - arr[2 * i + 1]];
const distance = (arr, i, j) => Math.sqrt(((arr[2 * i] - arr[2 * j]) ** 2) + ((arr[2 * i + 1] - arr[2 * j + 1]) ** 2));
// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
const ctlpts = (x1, y1, x2, y2, x3, y3) => {
// eslint-disable-next-line prefer-rest-params
const v = va(arguments, 0, 2);
// eslint-disable-next-line prefer-rest-params
const d01 = distance(arguments, 0, 1);
// eslint-disable-next-line prefer-rest-params
const d12 = distance(arguments, 1, 2);
const d012 = d01 + d12;
return [
x2 - v[0] * tension * d01 / d012, y2 - v[1] * tension * d01 / d012,
x2 + v[0] * tension * d12 / d012, y2 + v[1] * tension * d12 / d012,
];
};
const pts: any[] = [];
for (const pt of points) {
pts.push(pt[0]);
pts.push(pt[1]);
}
let cps = [];
for (let i = 0; i < pts.length - 2; i += 1) {
// @ts-ignore
cps = cps.concat(ctlpts(pts[2 * i + 0], pts[2 * i + 1], pts[2 * i + 2], pts[2 * i + 3], pts[2 * i + 4], pts[2 * i + 5]));
}
ctx.lineWidth = options.lineWidth;
ctx.strokeStyle = options.color;
if (points.length === 2) {
ctx.beginPath();
ctx.moveTo(pts[0], pts[1]);
ctx.lineTo(pts[2], pts[3]);
} else {
ctx.beginPath();
ctx.moveTo(pts[0], pts[1]);
// first segment is a quadratic
ctx.quadraticCurveTo(cps[0], cps[1], pts[2], pts[3]);
// for all middle points, connect with bezier
let i;
for (i = 2; i < ((pts.length / 2) - 1); i += 1) {
ctx.bezierCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], cps[(2 * (i - 1)) * 2], cps[(2 * (i - 1)) * 2 + 1], pts[i * 2], pts[i * 2 + 1]);
}
// last segment is a quadratic
ctx.quadraticCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], pts[i * 2], pts[i * 2 + 1]);
}
ctx.stroke();
} }
export async function gesture(inCanvas, result) { export async function gesture(inCanvas, result) {
@ -193,7 +116,9 @@ export async function face(inCanvas, result) {
ctx.font = options.font; ctx.font = options.font;
ctx.strokeStyle = options.color; ctx.strokeStyle = options.color;
ctx.fillStyle = options.color; ctx.fillStyle = options.color;
if (options.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); if (options.drawBoxes) {
rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]);
}
// silly hack since fillText does not suport new line // silly hack since fillText does not suport new line
const labels:string[] = []; const labels:string[] = [];
labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`); labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`);
@ -299,82 +224,70 @@ export async function body(inCanvas, result) {
} }
} }
if (options.drawPolygons) { if (options.drawPolygons) {
const path = new Path2D();
let root;
let part; let part;
const points: any[] = [];
// torso // torso
root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); points.length = 0;
if (root && root.score > config.body.scoreThreshold) { part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
const points: any[] = []; if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
points.push([root.position.x, root.position.y, 'leftShoulder']); part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHip');
part = result[i].keypoints.find((a) => a.part === 'rightHip'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHip');
part = result[i].keypoints.find((a) => a.part === 'leftHip'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); if (points.length === 5) lines(ctx, points); // only draw if we have complete torso
lines(ctx, points);
}
// leg left // leg left
root = result[i].keypoints.find((a) => a.part === 'leftHip'); points.length = 0;
if (root && root.score > config.body.scoreThreshold) { part = result[i].keypoints.find((a) => a.part === 'leftHip');
const points: any[] = []; if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftKnee');
part = result[i].keypoints.find((a) => a.part === 'leftKnee'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHeel');
part = result[i].keypoints.find((a) => a.part === 'leftHeel'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftFoot');
part = result[i].keypoints.find((a) => a.part === 'leftFoot'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); curves(ctx, points);
lines(ctx, points);
}
// leg right // leg right
root = result[i].keypoints.find((a) => a.part === 'rightHip'); points.length = 0;
if (root && root.score > config.body.scoreThreshold) { part = result[i].keypoints.find((a) => a.part === 'rightHip');
const points: any[] = []; if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightKnee');
part = result[i].keypoints.find((a) => a.part === 'rightKnee'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHeel');
part = result[i].keypoints.find((a) => a.part === 'rightHeel'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightFoot');
part = result[i].keypoints.find((a) => a.part === 'rightFoot'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); curves(ctx, points);
lines(ctx, points);
}
// arm left // arm left
root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); points.length = 0;
if (root && root.score > config.body.scoreThreshold) { part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
const points: any[] = []; if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftElbow');
part = result[i].keypoints.find((a) => a.part === 'leftElbow'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftWrist');
part = result[i].keypoints.find((a) => a.part === 'leftWrist'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftPalm');
part = result[i].keypoints.find((a) => a.part === 'leftPalm'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); curves(ctx, points);
lines(ctx, points);
}
// arm right // arm right
root = result[i].keypoints.find((a) => a.part === 'rightShoulder'); points.length = 0;
if (root && root.score > config.body.scoreThreshold) { part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
const points: any[] = []; if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightElbow');
part = result[i].keypoints.find((a) => a.part === 'rightElbow'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightWrist');
part = result[i].keypoints.find((a) => a.part === 'rightWrist'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightPalm');
part = result[i].keypoints.find((a) => a.part === 'rightPalm'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]);
if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); curves(ctx, points);
lines(ctx, points);
}
// draw all // draw all
ctx.stroke(path);
} }
} }
} }

16
src/human.ts
View File

@ -1,4 +1,5 @@
import { log } from './log'; import { log } from './log';
import * as sysinfo from './sysinfo';
import * as tf from '../dist/tfjs.esm.js'; import * as tf from '../dist/tfjs.esm.js';
import * as backend from './tfjs/backend'; import * as backend from './tfjs/backend';
import * as facemesh from './blazeface/facemesh'; import * as facemesh from './blazeface/facemesh';
@ -61,6 +62,7 @@ class Human {
emotion: any; emotion: any;
body: any; body: any;
hand: any; hand: any;
sysinfo: any;
constructor(userConfig = {}) { constructor(userConfig = {}) {
this.tf = tf; this.tf = tf;
@ -95,6 +97,8 @@ class Human {
this.emotion = emotion; this.emotion = emotion;
this.body = this.config.body.modelType.startsWith('posenet') ? posenet : blazepose; this.body = this.config.body.modelType.startsWith('posenet') ? posenet : blazepose;
this.hand = handpose; this.hand = handpose;
// include platform info
this.sysinfo = sysinfo.info();
} }
profile() { profile() {
@ -139,7 +143,11 @@ class Human {
if (userConfig) this.config = mergeDeep(this.config, userConfig); if (userConfig) this.config = mergeDeep(this.config, userConfig);
if (this.firstRun) { if (this.firstRun) {
if (this.config.debug) log(`version: ${this.version} TensorFlow/JS version: ${this.tf.version_core}`); if (this.config.debug) log(`version: ${this.version}`);
if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`);
if (this.config.debug) log('platform:', this.sysinfo.platform);
if (this.config.debug) log('agent:', this.sysinfo.agent);
await this.checkBackend(true); await this.checkBackend(true);
if (this.tf.ENV.flags.IS_BROWSER) { if (this.tf.ENV.flags.IS_BROWSER) {
if (this.config.debug) log('configuration:', this.config); if (this.config.debug) log('configuration:', this.config);
@ -206,9 +214,11 @@ class Human {
if (this.config.debug) log('setting backend:', this.config.backend); if (this.config.debug) log('setting backend:', this.config.backend);
if (this.config.backend === 'wasm') { if (this.config.backend === 'wasm') {
if (this.config.debug) log('settings wasm path:', this.config.wasmPath); if (this.config.debug) log('wasm path:', this.config.wasmPath);
this.tf.setWasmPaths(this.config.wasmPath); this.tf.setWasmPaths(this.config.wasmPath);
const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
const mt = await this.tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
if (this.config.debug) log(`wasm execution: ${simd ? 'SIMD' : 'no SIMD'} ${mt ? 'multithreaded' : 'singlethreaded'}`);
if (!simd) log('warning: wasm simd support is not enabled'); if (!simd) log('warning: wasm simd support is not enabled');
} }
@ -538,7 +548,7 @@ class Human {
else res = await this.warmupNode(); else res = await this.warmupNode();
this.config.videoOptimized = video; this.config.videoOptimized = video;
const t1 = now(); const t1 = now();
if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res); if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms');
return res; return res;
} }
} }

61
src/node.js
View File

@ -1,61 +0,0 @@
const console = require('console');
const process = require('process');
// const URL = require('url');
const tf = require('@tensorflow/tfjs-node');
const Human = require('../dist/human.node').default; // this resolves to project root which is '@vladmandic/human'
const logger = new console.Console({
stdout: process.stdout,
stderr: process.stderr,
ignoreErrors: true,
// groupIndentation: 2,
inspectOptions: {
showHidden: true,
depth: 5,
colors: true,
showProxy: true,
maxArrayLength: 1024,
maxStringLength: 10240,
breakLength: 300,
compact: 64,
sorted: false,
getters: true,
},
});
const config = {
backend: 'tensorflow',
videoOptimized: false,
face: {
detector: { modelPath: 'file://models/blazeface-back.json' },
mesh: { modelPath: 'file://models/facemesh.json' },
iris: { modelPath: 'file://models/iris.json' },
age: { modelPath: 'file://models/age-ssrnet-imdb.json' },
gender: { modelPath: 'file://models/gender.json' },
emotion: { modelPath: 'file://models/emotion.json' },
embedding: { modelPath: 'file://models/mobilefacenet.json' },
},
body: { modelPath: 'file://models/posenet.json' },
hand: {
detector: { modelPath: 'file://models/handdetect.json' },
skeleton: { modelPath: 'file://models/handskeleton.json' },
},
};
async function main() {
await tf.ready();
const human = new Human(config);
logger.info('Human:', human.version);
logger.info('Current folder:', process.env.PWD);
logger.info('Active Configuration', human.config);
logger.info('TFJS Version:', tf.version_core, 'Backend:', tf.getBackend());
logger.info('TFJS Flags:', tf.env().features);
logger.info('Loading models:');
await human.load(config);
const loaded = Object.keys(human.models).filter((a) => human.models[a]);
logger.info('Loaded:', loaded);
logger.info('Memory state:', human.tf.engine().memory());
logger.info('Test Complete');
}
main();

2
src/posenet/util.ts
View File

@ -44,7 +44,7 @@ export function scalePose(pose, scaleY, scaleX) {
keypoints: pose.keypoints.map(({ score, part, position }) => ({ keypoints: pose.keypoints.map(({ score, part, position }) => ({
score, score,
part, part,
position: { x: position.x * scaleX, y: position.y * scaleY }, position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) },
})), })),
}; };
} }

18
src/sysinfo.ts Normal file
View File

@ -0,0 +1,18 @@
export function info() {
let platform;
let agent;
if (typeof navigator !== 'undefined') {
const raw = navigator.userAgent.match(/\(([^()]+)\)/g);
if (raw && raw[0]) {
// @ts-ignore
platform = raw[0].match(/\(([^()]+)\)/g)[0].replace(/\(|\)/g, '');
agent = navigator.userAgent.replace(raw[0], '');
if (platform[1]) agent = agent.replace(raw[1], '');
agent = agent.replace(/ /g, ' ');
}
} else if (typeof process !== 'undefined') {
platform = `${process.platform} ${process.arch}`;
agent = `NodeJS ${process.version}`;
}
return { platform, agent };
}

1
types/draw.d.ts vendored
View File

@ -13,6 +13,7 @@ export declare const options: {
drawPolygons: boolean; drawPolygons: boolean;
fillPolygons: boolean; fillPolygons: boolean;
useDepth: boolean; useDepth: boolean;
useCurves: boolean;
bufferedOutput: boolean; bufferedOutput: boolean;
}; };
export declare function gesture(inCanvas: any, result: any): Promise<void>; export declare function gesture(inCanvas: any, result: any): Promise<void>;

1
types/human.d.ts vendored
View File

@ -19,6 +19,7 @@ declare class Human {
emotion: any; emotion: any;
body: any; body: any;
hand: any; hand: any;
sysinfo: any;
constructor(userConfig?: {}); constructor(userConfig?: {});
profile(): {}; profile(): {};
analyze(...msg: any[]): void; analyze(...msg: any[]): void;

4
types/sysinfo.d.ts vendored Normal file
View File

@ -0,0 +1,4 @@
export declare function info(): {
platform: any;
agent: any;
};

2
wiki

@ -1 +1 @@
Subproject commit ce4f3e12bd49ee404186c55ab977b4b1612d17f4 Subproject commit 562a698daecaecf8580120fc4e1c9b6ac66ba537