ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

optimized demos and added scoped runs

pull/50/head
Vladimir Mandic 2020-10-15 09:43:16 -04:00
parent b4cccc3c76
commit 1268fcef6f
22 changed files with 4214 additions and 71014 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.json
View File

@ -50,6 +50,7 @@
"promise/no-nesting": "off", "promise/no-nesting": "off",
"import/no-absolute-path": "off", "import/no-absolute-path": "off",
"import/no-extraneous-dependencies": "off", "import/no-extraneous-dependencies": "off",
"node/no-unpublished-import": "off",
"node/no-unpublished-require": "off", "node/no-unpublished-require": "off",
"no-regex-spaces": "off", "no-regex-spaces": "off",
"radix": "off" "radix": "off"

6
README.md
View File

@ -160,10 +160,8 @@ If your application resides in a different folder, modify `modelPath` property i
Demos are included in `/demo`: Demos are included in `/demo`:
Browser: Browser:
- `demo-esm`: Demo using Browser with ESM module - `demo-esm`: Full demo using Browser with ESM module, includes selectable backends and webworkers
- `demo-iife`: Demo using Browser with IIFE module - `demo-iife`: Older demo using Browser with IIFE module
- `demo-webworker`: Demo using Browser with ESM module and Web Workers
*All three following demos are identical, they just illustrate different ways to load and work with `Human` library:*
NodeJS: NodeJS:
- `demo-node`: Demo using NodeJS with CJS module - `demo-node`: Demo using NodeJS with CJS module

0
demo/demo-webworker-worker.js → demo/demo-esm-webworker.js
View File

4
demo/demo-esm.html
View File

@ -1,7 +1,7 @@
<head> <head>
<script src="../assets/quicksettings.js"></script> <script src="../assets/quicksettings.js"></script>
<script src="../assets/tf.min.js"></script> <!-- <script src="../assets/tf.min.js"></script> -->
<script src="../assets/tf-backend-wasm.min.js"></script> <!-- <script src="../assets/tf-backend-wasm.min.js"></script> -->
<script src="./demo-esm.js" type="module"></script> <script src="./demo-esm.js" type="module"></script>
</head> </head>
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-variant: small-caps"> <body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-variant: small-caps">

103
demo/demo-esm.js
View File

@ -3,7 +3,7 @@
import human from '../dist/human.esm.js'; import human from '../dist/human.esm.js';
const ui = { const ui = {
backend: 'wasm', backend: 'webgl',
baseColor: 'rgba(255, 200, 255, 0.3)', baseColor: 'rgba(255, 200, 255, 0.3)',
baseLabel: 'rgba(255, 200, 255, 0.8)', baseLabel: 'rgba(255, 200, 255, 0.8)',
baseFont: 'small-caps 1.2rem "Segoe UI"', baseFont: 'small-caps 1.2rem "Segoe UI"',
@ -24,6 +24,8 @@ const config = {
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
}; };
let settings; let settings;
let worker;
let timeStamp;
function str(...msg) { function str(...msg) {
if (!Array.isArray(msg)) return msg; if (!Array.isArray(msg)) return msg;
@ -35,13 +37,30 @@ function str(...msg) {
return line; return line;
} }
async function setupTF() { async function setupTF(input) {
// pause video if running before changing backend
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
if (live) await input.pause();
// if user explicitly loaded tfjs, override one used in human library
if (window.tf) human.tf = window.tf;
// cheks for wasm backend
if (ui.backend === 'wasm') { if (ui.backend === 'wasm') {
if (!window.tf) {
document.getElementById('log').innerText = 'Error: WASM Backend is not loaded, enable it in HTML file';
ui.backend = 'webgl';
} else {
human.tf = window.tf;
tf.env().set('WASM_HAS_SIMD_SUPPORT', false); tf.env().set('WASM_HAS_SIMD_SUPPORT', false);
tf.env().set('WASM_HAS_MULTITHREAD_SUPPORT', true); tf.env().set('WASM_HAS_MULTITHREAD_SUPPORT', true);
} }
}
await human.tf.setBackend(ui.backend); await human.tf.setBackend(ui.backend);
await human.tf.ready(); await human.tf.ready();
// continue video if it was previously running
if (live) await input.play();
} }
async function drawFace(result, canvas) { async function drawFace(result, canvas) {
@ -201,23 +220,9 @@ async function drawHand(result, canvas) {
} }
} }
async function runHumanDetect(input, canvas) { async function drawResults(input, result, canvas) {
const log = document.getElementById('log');
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
// perform detect if live video or not video at all
if (live || !(input instanceof HTMLVideoElement)) {
// perform detection
const t0 = performance.now();
let result;
try {
result = await human.detect(input, config);
} catch (err) {
log.innerText = err.message;
}
if (!result) return;
const t1 = performance.now();
// update fps // update fps
settings.setValue('FPS', Math.round(1000 / (t1 - t0))); settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
// draw image from video // draw image from video
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
@ -229,17 +234,50 @@ async function runHumanDetect(input, canvas) {
const engine = await human.tf.engine(); const engine = await human.tf.engine();
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`; const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
const gpu = engine.backendInstance.numBytesInGPU ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : ''; const gpu = engine.backendInstance.numBytesInGPU ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
const log = document.getElementById('log');
log.innerText = ` log.innerText = `
TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu} TFJS Version: ${human.tf.version_core} | Backend: {human.tf.getBackend()} | Memory: ${memory} ${gpu}
Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
`; `;
// rinse & repeate }
// if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes
async function webWorker(input, image, canvas) {
if (!worker) {
// create new webworker
worker = new Worker('demo-esm-webworker.js', { type: 'module' });
// after receiving message from webworker, parse&draw results and send new frame for processing
worker.addEventListener('message', async (msg) => {
await drawResults(input, msg.data, canvas);
// eslint-disable-next-line no-use-before-define
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
});
}
// const offscreen = image.transferControlToOffscreen();
worker.postMessage({ image, config });
}
async function runHumanDetect(input, canvas) {
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
timeStamp = performance.now();
// perform detect if live video or not video at all
if (live || !(input instanceof HTMLVideoElement)) {
if (settings.getValue('Use Web Worker')) {
// get image data from video as we cannot send html objects to webworker
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
const ctx = offscreen.getContext('2d');
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
// perform detection
await webWorker(input, data, canvas);
} else {
const result = await human.detect(input, config);
await drawResults(input, result, canvas);
if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
} }
}
} }
function setupGUI() { function setupUI(input) {
// add all variables to ui control panel // add all variables to ui control panel
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
const style = document.createElement('style'); const style = document.createElement('style');
@ -266,9 +304,9 @@ function setupGUI() {
} }
runHumanDetect(video, canvas); runHumanDetect(video, canvas);
}); });
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], (val) => { settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => {
ui.backend = val.value; ui.backend = val.value;
setupTF(); await setupTF(input);
}); });
settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title'); settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title');
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val); settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
@ -305,6 +343,7 @@ function setupGUI() {
config.hand.iouThreshold = parseFloat(val); config.hand.iouThreshold = parseFloat(val);
}); });
settings.addHTML('title', 'UI Options'); settings.hideTitle('title'); settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
settings.addBoolean('Use Web Worker', false);
settings.addBoolean('Draw Boxes', true); settings.addBoolean('Draw Boxes', true);
settings.addBoolean('Draw Points', true); settings.addBoolean('Draw Points', true);
settings.addBoolean('Draw Polygons', true); settings.addBoolean('Draw Polygons', true);
@ -357,17 +396,17 @@ async function setupImage() {
} }
async function main() { async function main() {
// initialize tensorflow
await setupTF();
// setup ui control panel
await setupGUI();
// setup webcam // setup webcam
const video = await setupCamera(); const input = await setupCamera();
// or setup image // or setup image
// const image = await setupImage(); // const input = await setupImage();
// setup output canvas from input object, select video or image // setup output canvas from input object
await setupCanvas(video); await setupCanvas(input);
// run actual detection. if input is video, it will run in a loop else it will run only once // run actual detection. if input is video, it will run in a loop else it will run only once
// setup ui control panel
await setupUI(input);
// initialize tensorflow
await setupTF(input);
// runHumanDetect(video, canvas); // runHumanDetect(video, canvas);
} }

7
demo/demo-iife.js
View File

@ -178,12 +178,7 @@ async function runHumanDetect(input, canvas) {
if (live || !(input instanceof HTMLVideoElement)) { if (live || !(input instanceof HTMLVideoElement)) {
// perform detection // perform detection
const t0 = performance.now(); const t0 = performance.now();
let result; const result = await human.detect(input, config);
try {
result = await human.detect(input, config);
} catch (err) {
log.innerText = err.message;
}
if (!result) return; if (!result) return;
const t1 = performance.now(); const t1 = performance.now();
// update fps // update fps

12
demo/demo-webworker.html
View File

@ -1,12 +0,0 @@
<head>
<script src="https://cdn.jsdelivr.net/npm/quicksettings"></script>
<script src="./demo-webworker.js" type="module"></script>
</head>
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'">
<div id="main">
<video id="video" playsinline style="display: none"></video>
<image id="image" src="" style="display: none"></video>
<canvas id="canvas"></canvas>
<div id="log">Starting Human library</div>
</div>
</body>

339
demo/demo-webworker.js
View File

@ -1,339 +0,0 @@
/* global QuickSettings */
import human from '../dist/human.esm.js';
const ui = {
baseColor: 'rgba(255, 200, 255, 0.3)',
baseFont: 'small-caps 1.2rem "Segoe UI"',
baseLineWidth: 16,
};
const config = {
face: {
enabled: true,
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
mesh: { enabled: true },
iris: { enabled: true },
age: { enabled: true, skipFrames: 5 },
gender: { enabled: true },
},
body: { enabled: true, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
hand: { enabled: true, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
};
let settings;
let worker;
let timeStamp;
async function drawFace(result, canvas) {
const ctx = canvas.getContext('2d');
ctx.fillStyle = ui.baseColor;
ctx.strokeStyle = ui.baseColor;
ctx.font = ui.baseFont;
for (const face of result) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
if (settings.getValue('Draw Boxes')) {
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
}
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
ctx.stroke();
ctx.lineWidth = 1;
if (face.mesh) {
if (settings.getValue('Draw Points')) {
for (const point of face.mesh) {
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
ctx.beginPath();
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
ctx.fill();
}
}
if (settings.getValue('Draw Polygons')) {
for (let i = 0; i < human.facemesh.triangulation.length / 3; i++) {
const points = [
human.facemesh.triangulation[i * 3 + 0],
human.facemesh.triangulation[i * 3 + 1],
human.facemesh.triangulation[i * 3 + 2],
].map((index) => face.mesh[index]);
const path = new Path2D();
path.moveTo(points[0][0], points[0][1]);
for (const point of points) {
path.lineTo(point[0], point[1]);
}
path.closePath();
ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
ctx.stroke(path);
if (settings.getValue('Fill Polygons')) {
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
ctx.fill(path);
}
}
}
}
}
}
async function drawBody(result, canvas) {
const ctx = canvas.getContext('2d');
ctx.fillStyle = ui.baseColor;
ctx.strokeStyle = ui.baseColor;
ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
for (const pose of result) {
if (settings.getValue('Draw Points')) {
for (const point of pose.keypoints) {
ctx.beginPath();
ctx.arc(point.position.x, point.position.y, 2, 0, 2 * Math.PI);
ctx.fill();
}
}
if (settings.getValue('Draw Polygons')) {
const path = new Path2D();
let part;
// torso
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
path.moveTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightHip');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftHip');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
path.lineTo(part.position.x, part.position.y);
// legs
part = pose.keypoints.find((a) => a.part === 'leftHip');
path.moveTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftKnee');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftAnkle');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightHip');
path.moveTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightKnee');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightAnkle');
path.lineTo(part.position.x, part.position.y);
// arms
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
path.moveTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftElbow');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'leftWrist');
path.lineTo(part.position.x, part.position.y);
// arms
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
path.moveTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightElbow');
path.lineTo(part.position.x, part.position.y);
part = pose.keypoints.find((a) => a.part === 'rightWrist');
path.lineTo(part.position.x, part.position.y);
// draw all
ctx.stroke(path);
}
}
}
async function drawHand(result, canvas) {
const ctx = canvas.getContext('2d');
ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
window.result = result;
for (const hand of result) {
if (settings.getValue('Draw Boxes')) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
ctx.stroke();
}
if (settings.getValue('Draw Points')) {
for (const point of hand.landmarks) {
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
ctx.beginPath();
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
ctx.fill();
}
}
if (settings.getValue('Draw Polygons')) {
const addPart = (part) => {
for (let i = 1; i < part.length; i++) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`;
ctx.moveTo(part[i - 1][0], part[i - 1][1]);
ctx.lineTo(part[i][0], part[i][1]);
ctx.stroke();
}
};
addPart(hand.annotations.indexFinger);
addPart(hand.annotations.middleFinger);
addPart(hand.annotations.ringFinger);
addPart(hand.annotations.pinky);
addPart(hand.annotations.thumb);
addPart(hand.annotations.palmBase);
}
}
}
async function drawResults(input, result, canvas) {
// draw image
const ctx = canvas.getContext('2d');
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
// update fps
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
// draw all results
drawFace(result.face, canvas);
drawBody(result.body, canvas);
drawHand(result.hand, canvas);
// update log
const engine = await human.tf.engine();
const log = document.getElementById('log');
log.innerText = `
TFJS Version: ${human.tf.version_core} Memory: ${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors
GPU Memory: used ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes free ${Math.floor(1024 * 1024 * engine.backendInstance.numMBBeforeWarning).toLocaleString()} bytes
Result: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes
`;
}
async function webWorker(input, image, canvas) {
if (!worker) {
// create new webworker
worker = new Worker('demo-webworker-worker.js', { type: 'module' });
// after receiving message from webworker, parse&draw results and send new frame for processing
worker.addEventListener('message', (msg) => {
drawResults(input, msg.data, canvas);
// eslint-disable-next-line no-use-before-define
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
});
}
timeStamp = performance.now();
// const offscreen = image.transferControlToOffscreen();
worker.postMessage({ image, config });
}
async function runHumanDetect(input, canvas) {
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
// perform detect if live video or not video at all
if (live || !(input instanceof HTMLVideoElement)) {
// get image data from video as we cannot send html objects to webworker
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
const ctx = offscreen.getContext('2d');
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
// perform detection
webWorker(input, data, canvas);
}
}
function setupGUI() {
// add all variables to ui control panel
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
settings.addRange('FPS', 0, 100, 0, 1);
settings.addBoolean('Pause', false, (val) => {
const video = document.getElementById('video');
const canvas = document.getElementById('canvas');
if (val) video.pause();
else video.play();
runHumanDetect(video, canvas);
});
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
settings.addBoolean('Draw Boxes', false);
settings.addBoolean('Draw Points', true);
settings.addBoolean('Draw Polygons', true);
settings.addBoolean('Fill Polygons', true);
settings.bindText('baseColor', ui.baseColor, config);
settings.bindText('baseFont', ui.baseFont, config);
settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config);
settings.addHTML('line2', '<hr>'); settings.hideTitle('line2');
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
settings.addHTML('line3', '<hr>'); settings.hideTitle('line3');
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
config.face.detector.maxFaces = parseInt(val);
config.body.maxDetections = parseInt(val);
});
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
config.face.detector.skipFrames = parseInt(val);
config.face.age.skipFrames = parseInt(val);
config.hand.skipFrames = parseInt(val);
});
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
config.face.detector.minConfidence = parseFloat(val);
config.hand.minConfidence = parseFloat(val);
});
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
config.face.detector.scoreThreshold = parseFloat(val);
config.hand.scoreThreshold = parseFloat(val);
config.body.scoreThreshold = parseFloat(val);
});
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
config.face.detector.iouThreshold = parseFloat(val);
config.hand.iouThreshold = parseFloat(val);
});
}
async function setupCanvas(input) {
// setup canvas object to same size as input as camera resolution may change
const canvas = document.getElementById('canvas');
canvas.width = input.width;
canvas.height = input.height;
return canvas;
}
// eslint-disable-next-line no-unused-vars
async function setupCamera() {
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
const video = document.getElementById('video');
if (!navigator.mediaDevices) {
document.getElementById('log').innerText = 'Video not supported';
return null;
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight },
});
video.srcObject = stream;
return new Promise((resolve) => {
video.onloadedmetadata = () => {
video.width = video.videoWidth;
video.height = video.videoHeight;
video.play();
resolve(video);
};
});
}
// eslint-disable-next-line no-unused-vars
async function setupImage() {
const image = document.getElementById('image');
image.width = window.innerWidth;
image.height = window.innerHeight;
return new Promise((resolve) => {
image.onload = () => resolve(image);
image.src = 'sample.jpg';
});
}
async function main() {
// initialize tensorflow
await human.tf.setBackend('webgl');
await human.tf.ready();
// setup ui control panel
await setupGUI();
// setup webcam
const video = await setupCamera();
// or setup image
// const image = await setupImage();
// setup output canvas from input object, select video or image
const canvas = await setupCanvas(video);
// run actual detection. if input is video, it will run in a loop else it will run only once
runHumanDetect(video, canvas);
}
window.onload = main;
window.onresize = main;

49
dist/human-nobundle.cjs vendored
View File

@ -3887,21 +3887,20 @@ var require_ssrnet = __commonJS((exports2) => {
let last = {age: 0, gender: ""}; let last = {age: 0, gender: ""};
let frame = 0; let frame = 0;
async function getImage(image, size) { async function getImage(image, size) {
const tensor = tf2.tidy(() => {
const buffer = tf2.browser.fromPixels(image); const buffer = tf2.browser.fromPixels(image);
const resize = tf2.image.resizeBilinear(buffer, [size, size]); const resize = tf2.image.resizeBilinear(buffer, [size, size]);
const expand = tf2.cast(tf2.expandDims(resize, 0), "float32"); const expand = tf2.cast(tf2.expandDims(resize, 0), "float32");
return expand; return expand;
});
return tensor;
} }
async function loadAge(config) { async function loadAge(config) {
if (!models2.age) if (!models2.age)
models2.age = await tf2.loadGraphModel(config.face.age.modelPath); models2.age = await tf2.loadGraphModel(config.face.age.modelPath);
return models2.age;
} }
async function loadGender(config) { async function loadGender(config) {
if (!models2.gender) if (!models2.gender)
models2.gender = await tf2.loadGraphModel(config.face.gender.modelPath); models2.gender = await tf2.loadGraphModel(config.face.gender.modelPath);
return models2.gender;
} }
async function predict(image, config) { async function predict(image, config) {
frame += 1; frame += 1;
@ -3959,6 +3958,7 @@ var require_emotion = __commonJS((exports2) => {
async function load(config) { async function load(config) {
if (!models2.emotion) if (!models2.emotion)
models2.emotion = await tf2.loadGraphModel(config.face.emotion.modelPath); models2.emotion = await tf2.loadGraphModel(config.face.emotion.modelPath);
return models2.emotion;
} }
async function predict(image, config) { async function predict(image, config) {
frame += 1; frame += 1;
@ -5142,9 +5142,12 @@ const handpose = require_handpose();
const defaults = require_config().default; const defaults = require_config().default;
const models = { const models = {
facemesh: null, facemesh: null,
blazeface: null, posenet: null,
ssrnet: null, handpose: null,
iris: null iris: null,
age: null,
gender: null,
emotion: null
}; };
function mergeDeep(...objects) { function mergeDeep(...objects) {
const isObject = (obj) => obj && typeof obj === "object"; const isObject = (obj) => obj && typeof obj === "object";
@ -5166,19 +5169,18 @@ function mergeDeep(...objects) {
async function detect(input, userConfig) { async function detect(input, userConfig) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const config = mergeDeep(defaults, userConfig); const config = mergeDeep(defaults, userConfig);
if (config.face.age.enabled) if (config.face.enabled && !models.facemesh)
await ssrnet.loadAge(config); models.facemesh = await facemesh.load(config.face);
if (config.face.gender.enabled)
await ssrnet.loadGender(config);
if (config.face.emotion.enabled)
await emotion.load(config);
if (config.body.enabled && !models.posenet) if (config.body.enabled && !models.posenet)
models.posenet = await posenet.load(config.body); models.posenet = await posenet.load(config.body);
if (config.hand.enabled && !models.handpose) if (config.hand.enabled && !models.handpose)
models.handpose = await handpose.load(config.hand); models.handpose = await handpose.load(config.hand);
if (config.face.enabled && !models.facemesh) if (config.face.enabled && config.face.age.enabled && !models.age)
models.facemesh = await facemesh.load(config.face); models.age = await ssrnet.loadAge(config);
tf.engine().startScope(); if (config.face.enabled && config.face.gender.enabled && !models.gender)
models.gender = await ssrnet.loadGender(config);
if (config.face.enabled && config.face.emotion.enabled && !models.emotion)
models.emotion = await emotion.load(config);
let savedWebglPackDepthwiseConvFlag; let savedWebglPackDepthwiseConvFlag;
if (tf.getBackend() === "webgl") { if (tf.getBackend() === "webgl") {
savedWebglPackDepthwiseConvFlag = tf.env().get("WEBGL_PACK_DEPTHWISECONV"); savedWebglPackDepthwiseConvFlag = tf.env().get("WEBGL_PACK_DEPTHWISECONV");
@ -5188,25 +5190,30 @@ async function detect(input, userConfig) {
let timeStamp; let timeStamp;
timeStamp = performance.now(); timeStamp = performance.now();
let poseRes = []; let poseRes = [];
tf.engine().startScope();
if (config.body.enabled) if (config.body.enabled)
poseRes = await models.posenet.estimatePoses(input, config.body); poseRes = await models.posenet.estimatePoses(input, config.body);
tf.engine().endScope();
perf.body = Math.trunc(performance.now() - timeStamp); perf.body = Math.trunc(performance.now() - timeStamp);
timeStamp = performance.now(); timeStamp = performance.now();
let handRes = []; let handRes = [];
tf.engine().startScope();
if (config.hand.enabled) if (config.hand.enabled)
handRes = await models.handpose.estimateHands(input, config.hand); handRes = await models.handpose.estimateHands(input, config.hand);
tf.engine().endScope();
perf.hand = Math.trunc(performance.now() - timeStamp); perf.hand = Math.trunc(performance.now() - timeStamp);
const faceRes = []; const faceRes = [];
if (config.face.enabled) { if (config.face.enabled) {
timeStamp = performance.now(); timeStamp = performance.now();
tf.engine().startScope();
const faces = await models.facemesh.estimateFaces(input, config.face); const faces = await models.facemesh.estimateFaces(input, config.face);
perf.face = Math.trunc(performance.now() - timeStamp); perf.face = Math.trunc(performance.now() - timeStamp);
for (const face of faces) { for (const face of faces) {
timeStamp = performance.now(); timeStamp = performance.now();
const ssrdata = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {}; const ssrData = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
perf.agegender = Math.trunc(performance.now() - timeStamp); perf.agegender = Math.trunc(performance.now() - timeStamp);
timeStamp = performance.now(); timeStamp = performance.now();
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {}; const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
perf.emotion = Math.trunc(performance.now() - timeStamp); perf.emotion = Math.trunc(performance.now() - timeStamp);
face.image.dispose(); face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0; const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -5215,15 +5222,15 @@ async function detect(input, userConfig) {
box: face.box, box: face.box,
mesh: face.mesh, mesh: face.mesh,
annotations: face.annotations, annotations: face.annotations,
age: ssrdata.age, age: ssrData.age,
gender: ssrdata.gender, gender: ssrData.gender,
emotion: emotiondata, emotion: emotionData,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0 iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
}); });
} }
tf.engine().endScope();
} }
tf.env().set("WEBGL_PACK_DEPTHWISECONV", savedWebglPackDepthwiseConvFlag); tf.env().set("WEBGL_PACK_DEPTHWISECONV", savedWebglPackDepthwiseConvFlag);
tf.engine().endScope();
perf.total = Object.values(perf).reduce((a, b) => a + b); perf.total = Object.values(perf).reduce((a, b) => a + b);
resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf}); resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf});
}); });

4
dist/human-nobundle.cjs.map vendored
View File

File diff suppressed because one or more lines are too long

188
dist/human.cjs vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.cjs.map vendored
View File

File diff suppressed because one or more lines are too long

2
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

74062
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

188
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
package.json
View File

@ -37,7 +37,7 @@
"start": "node --trace-warnings --trace-uncaught --no-deprecation demo/demo-node.js", "start": "node --trace-warnings --trace-uncaught --no-deprecation demo/demo-node.js",
"lint": "eslint src/*.js demo/*.js", "lint": "eslint src/*.js demo/*.js",
"build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --outfile=dist/human.js src/index.js", "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --outfile=dist/human.js src/index.js",
"build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --external:fs --outfile=dist/human.esm.js src/index.js", "build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --outfile=dist/human.esm.js src/index.js",
"build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --outfile=dist/human.esm-nobundle.js src/index.js", "build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --outfile=dist/human.esm-nobundle.js src/index.js",
"build-node-bundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --minify --outfile=dist/human.cjs src/index.js", "build-node-bundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --minify --outfile=dist/human.cjs src/index.js",
"build-node-nobundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --outfile=dist/human-nobundle.cjs src/index.js", "build-node-nobundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --outfile=dist/human-nobundle.cjs src/index.js",

3
src/emotion/emotion.js
View File

@ -18,6 +18,7 @@ function getImage(image, size) {
async function load(config) { async function load(config) {
if (!models.emotion) models.emotion = await tf.loadGraphModel(config.face.emotion.modelPath); if (!models.emotion) models.emotion = await tf.loadGraphModel(config.face.emotion.modelPath);
return models.emotion;
} }
async function predict(image, config) { async function predict(image, config) {
@ -31,7 +32,7 @@ async function predict(image, config) {
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false); const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
const [r, g, b] = tf.split(resize, 3, 3); const [r, g, b] = tf.split(resize, 3, 3);
if (config.face.emotion.useGrayscale) { if (config.face.emotion.useGrayscale) {
// 0.2989 * R + 0.5870 * G + 0.1140 * B // https://www.mathworks.com/help/matlab/ref/rgb2gray.html // weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const r1 = tf.mul(r, [0.2989]); const r1 = tf.mul(r, [0.2989]);
const g1 = tf.mul(g, [0.5870]); const g1 = tf.mul(g, [0.5870]);
const b1 = tf.mul(b, [0.1140]); const b1 = tf.mul(b, [0.1140]);

40
src/index.js
View File

@ -6,13 +6,18 @@ const posenet = require('./posenet/posenet.js');
const handpose = require('./handpose/handpose.js'); const handpose = require('./handpose/handpose.js');
const defaults = require('./config.js').default; const defaults = require('./config.js').default;
// object that contains all initialized models
const models = { const models = {
facemesh: null, facemesh: null,
blazeface: null, posenet: null,
ssrnet: null, handpose: null,
iris: null, iris: null,
age: null,
gender: null,
emotion: null,
}; };
// helper function that performs deep merge of multiple objects so it allows full inheriance with overrides
function mergeDeep(...objects) { function mergeDeep(...objects) {
const isObject = (obj) => obj && typeof obj === 'object'; const isObject = (obj) => obj && typeof obj === 'object';
return objects.reduce((prev, obj) => { return objects.reduce((prev, obj) => {
@ -37,15 +42,14 @@ async function detect(input, userConfig) {
const config = mergeDeep(defaults, userConfig); const config = mergeDeep(defaults, userConfig);
// load models if enabled // load models if enabled
if (config.face.age.enabled) await ssrnet.loadAge(config); if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face);
if (config.face.gender.enabled) await ssrnet.loadGender(config);
if (config.face.emotion.enabled) await emotion.load(config);
if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body); if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body);
if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand); if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand);
if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face); if (config.face.enabled && config.face.age.enabled && !models.age) models.age = await ssrnet.loadAge(config);
if (config.face.enabled && config.face.gender.enabled && !models.gender) models.gender = await ssrnet.loadGender(config);
tf.engine().startScope(); if (config.face.enabled && config.face.emotion.enabled && !models.emotion) models.emotion = await emotion.load(config);
// explictly enable depthwiseconv since it's diasabled by default due to issues with large shaders
let savedWebglPackDepthwiseConvFlag; let savedWebglPackDepthwiseConvFlag;
if (tf.getBackend() === 'webgl') { if (tf.getBackend() === 'webgl') {
savedWebglPackDepthwiseConvFlag = tf.env().get('WEBGL_PACK_DEPTHWISECONV'); savedWebglPackDepthwiseConvFlag = tf.env().get('WEBGL_PACK_DEPTHWISECONV');
@ -58,29 +62,34 @@ async function detect(input, userConfig) {
// run posenet // run posenet
timeStamp = performance.now(); timeStamp = performance.now();
let poseRes = []; let poseRes = [];
tf.engine().startScope();
if (config.body.enabled) poseRes = await models.posenet.estimatePoses(input, config.body); if (config.body.enabled) poseRes = await models.posenet.estimatePoses(input, config.body);
tf.engine().endScope();
perf.body = Math.trunc(performance.now() - timeStamp); perf.body = Math.trunc(performance.now() - timeStamp);
// run handpose // run handpose
timeStamp = performance.now(); timeStamp = performance.now();
let handRes = []; let handRes = [];
tf.engine().startScope();
if (config.hand.enabled) handRes = await models.handpose.estimateHands(input, config.hand); if (config.hand.enabled) handRes = await models.handpose.estimateHands(input, config.hand);
tf.engine().endScope();
perf.hand = Math.trunc(performance.now() - timeStamp); perf.hand = Math.trunc(performance.now() - timeStamp);
// run facemesh, includes blazeface and iris // run facemesh, includes blazeface and iris
const faceRes = []; const faceRes = [];
if (config.face.enabled) { if (config.face.enabled) {
timeStamp = performance.now(); timeStamp = performance.now();
tf.engine().startScope();
const faces = await models.facemesh.estimateFaces(input, config.face); const faces = await models.facemesh.estimateFaces(input, config.face);
perf.face = Math.trunc(performance.now() - timeStamp); perf.face = Math.trunc(performance.now() - timeStamp);
for (const face of faces) { for (const face of faces) {
// run ssr-net age & gender, inherits face from blazeface // run ssr-net age & gender, inherits face from blazeface
timeStamp = performance.now(); timeStamp = performance.now();
const ssrdata = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {}; const ssrData = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
perf.agegender = Math.trunc(performance.now() - timeStamp); perf.agegender = Math.trunc(performance.now() - timeStamp);
// run emotion, inherits face from blazeface // run emotion, inherits face from blazeface
timeStamp = performance.now(); timeStamp = performance.now();
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {}; const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
perf.emotion = Math.trunc(performance.now() - timeStamp); perf.emotion = Math.trunc(performance.now() - timeStamp);
face.image.dispose(); face.image.dispose();
// calculate iris distance // calculate iris distance
@ -93,18 +102,19 @@ async function detect(input, userConfig) {
box: face.box, box: face.box,
mesh: face.mesh, mesh: face.mesh,
annotations: face.annotations, annotations: face.annotations,
age: ssrdata.age, age: ssrData.age,
gender: ssrdata.gender, gender: ssrData.gender,
emotion: emotiondata, emotion: emotionData,
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0, iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
}); });
} }
tf.engine().endScope();
} }
// set depthwiseconv to original value
tf.env().set('WEBGL_PACK_DEPTHWISECONV', savedWebglPackDepthwiseConvFlag); tf.env().set('WEBGL_PACK_DEPTHWISECONV', savedWebglPackDepthwiseConvFlag);
tf.engine().endScope(); // combine and return results
// combine results
perf.total = Object.values(perf).reduce((a, b) => a + b); perf.total = Object.values(perf).reduce((a, b) => a + b);
resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf }); resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf });
}); });

6
src/ssrnet/ssrnet.js
View File

@ -5,22 +5,20 @@ let last = { age: 0, gender: '' };
let frame = 0; let frame = 0;
async function getImage(image, size) { async function getImage(image, size) {
const tensor = tf.tidy(() => {
const buffer = tf.browser.fromPixels(image); const buffer = tf.browser.fromPixels(image);
const resize = tf.image.resizeBilinear(buffer, [size, size]); const resize = tf.image.resizeBilinear(buffer, [size, size]);
const expand = tf.cast(tf.expandDims(resize, 0), 'float32'); const expand = tf.cast(tf.expandDims(resize, 0), 'float32');
// const normalize = tf.mul(expand, [1.0 / 1.0]);
return expand; return expand;
});
return tensor;
} }
async function loadAge(config) { async function loadAge(config) {
if (!models.age) models.age = await tf.loadGraphModel(config.face.age.modelPath); if (!models.age) models.age = await tf.loadGraphModel(config.face.age.modelPath);
return models.age;
} }
async function loadGender(config) { async function loadGender(config) {
if (!models.gender) models.gender = await tf.loadGraphModel(config.face.gender.modelPath); if (!models.gender) models.gender = await tf.loadGraphModel(config.face.gender.modelPath);
return models.gender;
} }
async function predict(image, config) { async function predict(image, config) {