mirror of https://github.com/vladmandic/human
optimized demos and added scoped runs
Vladimir Mandic
parent
b4cccc3c76
commit
1268fcef6f
|
|
@ -50,6 +50,7 @@
|
|||
"promise/no-nesting": "off",
|
||||
"import/no-absolute-path": "off",
|
||||
"import/no-extraneous-dependencies": "off",
|
||||
"node/no-unpublished-import": "off",
|
||||
"node/no-unpublished-require": "off",
|
||||
"no-regex-spaces": "off",
|
||||
"radix": "off"
|
||||
|
|
|
|||
|
|
@ -160,10 +160,8 @@ If your application resides in a different folder, modify `modelPath` property i
|
|||
Demos are included in `/demo`:
|
||||
|
||||
Browser:
|
||||
- `demo-esm`: Demo using Browser with ESM module
|
||||
- `demo-iife`: Demo using Browser with IIFE module
|
||||
- `demo-webworker`: Demo using Browser with ESM module and Web Workers
|
||||
*All three following demos are identical, they just illustrate different ways to load and work with `Human` library:*
|
||||
- `demo-esm`: Full demo using Browser with ESM module, includes selectable backends and webworkers
|
||||
- `demo-iife`: Older demo using Browser with IIFE module
|
||||
|
||||
NodeJS:
|
||||
- `demo-node`: Demo using NodeJS with CJS module
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<head>
|
||||
<script src="../assets/quicksettings.js"></script>
|
||||
<script src="../assets/tf.min.js"></script>
|
||||
<script src="../assets/tf-backend-wasm.min.js"></script>
|
||||
<!-- <script src="../assets/tf.min.js"></script> -->
|
||||
<!-- <script src="../assets/tf-backend-wasm.min.js"></script> -->
|
||||
<script src="./demo-esm.js" type="module"></script>
|
||||
</head>
|
||||
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-variant: small-caps">
|
||||
|
|
|
|||
129
demo/demo-esm.js
129
demo/demo-esm.js
|
|
@ -3,7 +3,7 @@
|
|||
import human from '../dist/human.esm.js';
|
||||
|
||||
const ui = {
|
||||
backend: 'wasm',
|
||||
backend: 'webgl',
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseLabel: 'rgba(255, 200, 255, 0.8)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
|
|
@ -24,6 +24,8 @@ const config = {
|
|||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
};
|
||||
let settings;
|
||||
let worker;
|
||||
let timeStamp;
|
||||
|
||||
function str(...msg) {
|
||||
if (!Array.isArray(msg)) return msg;
|
||||
|
|
@ -35,13 +37,30 @@ function str(...msg) {
|
|||
return line;
|
||||
}
|
||||
|
||||
async function setupTF() {
|
||||
async function setupTF(input) {
|
||||
// pause video if running before changing backend
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
if (live) await input.pause();
|
||||
|
||||
// if user explicitly loaded tfjs, override one used in human library
|
||||
if (window.tf) human.tf = window.tf;
|
||||
|
||||
// cheks for wasm backend
|
||||
if (ui.backend === 'wasm') {
|
||||
tf.env().set('WASM_HAS_SIMD_SUPPORT', false);
|
||||
tf.env().set('WASM_HAS_MULTITHREAD_SUPPORT', true);
|
||||
if (!window.tf) {
|
||||
document.getElementById('log').innerText = 'Error: WASM Backend is not loaded, enable it in HTML file';
|
||||
ui.backend = 'webgl';
|
||||
} else {
|
||||
human.tf = window.tf;
|
||||
tf.env().set('WASM_HAS_SIMD_SUPPORT', false);
|
||||
tf.env().set('WASM_HAS_MULTITHREAD_SUPPORT', true);
|
||||
}
|
||||
}
|
||||
await human.tf.setBackend(ui.backend);
|
||||
await human.tf.ready();
|
||||
|
||||
// continue video if it was previously running
|
||||
if (live) await input.play();
|
||||
}
|
||||
|
||||
async function drawFace(result, canvas) {
|
||||
|
|
@ -201,45 +220,64 @@ async function drawHand(result, canvas) {
|
|||
}
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
async function drawResults(input, result, canvas) {
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
|
||||
// draw image from video
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
// draw all results
|
||||
drawFace(result.face, canvas);
|
||||
drawBody(result.body, canvas);
|
||||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
|
||||
const gpu = engine.backendInstance.numBytesInGPU ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
|
||||
const log = document.getElementById('log');
|
||||
log.innerText = `
|
||||
TFJS Version: ${human.tf.version_core} | Backend: {human.tf.getBackend()} | Memory: ${memory} ${gpu}
|
||||
Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
|
||||
`;
|
||||
}
|
||||
|
||||
async function webWorker(input, image, canvas) {
|
||||
if (!worker) {
|
||||
// create new webworker
|
||||
worker = new Worker('demo-esm-webworker.js', { type: 'module' });
|
||||
// after receiving message from webworker, parse&draw results and send new frame for processing
|
||||
worker.addEventListener('message', async (msg) => {
|
||||
await drawResults(input, msg.data, canvas);
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
});
|
||||
}
|
||||
// const offscreen = image.transferControlToOffscreen();
|
||||
worker.postMessage({ image, config });
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
timeStamp = performance.now();
|
||||
// perform detect if live video or not video at all
|
||||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
// perform detection
|
||||
const t0 = performance.now();
|
||||
let result;
|
||||
try {
|
||||
result = await human.detect(input, config);
|
||||
} catch (err) {
|
||||
log.innerText = err.message;
|
||||
if (settings.getValue('Use Web Worker')) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
// perform detection
|
||||
await webWorker(input, data, canvas);
|
||||
} else {
|
||||
const result = await human.detect(input, config);
|
||||
await drawResults(input, result, canvas);
|
||||
if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
}
|
||||
if (!result) return;
|
||||
const t1 = performance.now();
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (t1 - t0)));
|
||||
// draw image from video
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
// draw all results
|
||||
drawFace(result.face, canvas);
|
||||
drawBody(result.body, canvas);
|
||||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
|
||||
const gpu = engine.backendInstance.numBytesInGPU ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
|
||||
log.innerText = `
|
||||
TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu}
|
||||
Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
|
||||
`;
|
||||
// rinse & repeate
|
||||
// if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes
|
||||
if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
}
|
||||
}
|
||||
|
||||
function setupGUI() {
|
||||
function setupUI(input) {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
const style = document.createElement('style');
|
||||
|
|
@ -266,9 +304,9 @@ function setupGUI() {
|
|||
}
|
||||
runHumanDetect(video, canvas);
|
||||
});
|
||||
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], (val) => {
|
||||
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => {
|
||||
ui.backend = val.value;
|
||||
setupTF();
|
||||
await setupTF(input);
|
||||
});
|
||||
settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title');
|
||||
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
|
||||
|
|
@ -305,6 +343,7 @@ function setupGUI() {
|
|||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
|
||||
settings.addBoolean('Use Web Worker', false);
|
||||
settings.addBoolean('Draw Boxes', true);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
|
|
@ -357,17 +396,17 @@ async function setupImage() {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
// initialize tensorflow
|
||||
await setupTF();
|
||||
// setup ui control panel
|
||||
await setupGUI();
|
||||
// setup webcam
|
||||
const video = await setupCamera();
|
||||
const input = await setupCamera();
|
||||
// or setup image
|
||||
// const image = await setupImage();
|
||||
// setup output canvas from input object, select video or image
|
||||
await setupCanvas(video);
|
||||
// const input = await setupImage();
|
||||
// setup output canvas from input object
|
||||
await setupCanvas(input);
|
||||
// run actual detection. if input is video, it will run in a loop else it will run only once
|
||||
// setup ui control panel
|
||||
await setupUI(input);
|
||||
// initialize tensorflow
|
||||
await setupTF(input);
|
||||
// runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -178,12 +178,7 @@ async function runHumanDetect(input, canvas) {
|
|||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
// perform detection
|
||||
const t0 = performance.now();
|
||||
let result;
|
||||
try {
|
||||
result = await human.detect(input, config);
|
||||
} catch (err) {
|
||||
log.innerText = err.message;
|
||||
}
|
||||
const result = await human.detect(input, config);
|
||||
if (!result) return;
|
||||
const t1 = performance.now();
|
||||
// update fps
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
<head>
|
||||
<script src="https://cdn.jsdelivr.net/npm/quicksettings"></script>
|
||||
<script src="./demo-webworker.js" type="module"></script>
|
||||
</head>
|
||||
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'">
|
||||
<div id="main">
|
||||
<video id="video" playsinline style="display: none"></video>
|
||||
<image id="image" src="" style="display: none"></video>
|
||||
<canvas id="canvas"></canvas>
|
||||
<div id="log">Starting Human library</div>
|
||||
</div>
|
||||
</body>
|
||||
|
|
@ -1,339 +0,0 @@
|
|||
/* global QuickSettings */
|
||||
|
||||
import human from '../dist/human.esm.js';
|
||||
|
||||
const ui = {
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
baseLineWidth: 16,
|
||||
};
|
||||
|
||||
const config = {
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 5 },
|
||||
gender: { enabled: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
};
|
||||
let settings;
|
||||
let worker;
|
||||
let timeStamp;
|
||||
|
||||
async function drawFace(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
for (const face of result) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
|
||||
}
|
||||
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
|
||||
ctx.stroke();
|
||||
ctx.lineWidth = 1;
|
||||
if (face.mesh) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of face.mesh) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
for (let i = 0; i < human.facemesh.triangulation.length / 3; i++) {
|
||||
const points = [
|
||||
human.facemesh.triangulation[i * 3 + 0],
|
||||
human.facemesh.triangulation[i * 3 + 1],
|
||||
human.facemesh.triangulation[i * 3 + 2],
|
||||
].map((index) => face.mesh[index]);
|
||||
const path = new Path2D();
|
||||
path.moveTo(points[0][0], points[0][1]);
|
||||
for (const point of points) {
|
||||
path.lineTo(point[0], point[1]);
|
||||
}
|
||||
path.closePath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.stroke(path);
|
||||
if (settings.getValue('Fill Polygons')) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.fill(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawBody(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
for (const pose of result) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of pose.keypoints) {
|
||||
ctx.beginPath();
|
||||
ctx.arc(point.position.x, point.position.y, 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const path = new Path2D();
|
||||
let part;
|
||||
// torso
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// legs
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// draw all
|
||||
ctx.stroke(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawHand(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
window.result = result;
|
||||
for (const hand of result) {
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
|
||||
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
|
||||
ctx.stroke();
|
||||
}
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of hand.landmarks) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const addPart = (part) => {
|
||||
for (let i = 1; i < part.length; i++) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`;
|
||||
ctx.moveTo(part[i - 1][0], part[i - 1][1]);
|
||||
ctx.lineTo(part[i][0], part[i][1]);
|
||||
ctx.stroke();
|
||||
}
|
||||
};
|
||||
addPart(hand.annotations.indexFinger);
|
||||
addPart(hand.annotations.middleFinger);
|
||||
addPart(hand.annotations.ringFinger);
|
||||
addPart(hand.annotations.pinky);
|
||||
addPart(hand.annotations.thumb);
|
||||
addPart(hand.annotations.palmBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawResults(input, result, canvas) {
|
||||
// draw image
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
|
||||
// draw all results
|
||||
drawFace(result.face, canvas);
|
||||
drawBody(result.body, canvas);
|
||||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
const log = document.getElementById('log');
|
||||
log.innerText = `
|
||||
TFJS Version: ${human.tf.version_core} Memory: ${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors
|
||||
GPU Memory: used ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes free ${Math.floor(1024 * 1024 * engine.backendInstance.numMBBeforeWarning).toLocaleString()} bytes
|
||||
Result: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes
|
||||
`;
|
||||
}
|
||||
|
||||
async function webWorker(input, image, canvas) {
|
||||
if (!worker) {
|
||||
// create new webworker
|
||||
worker = new Worker('demo-webworker-worker.js', { type: 'module' });
|
||||
// after receiving message from webworker, parse&draw results and send new frame for processing
|
||||
worker.addEventListener('message', (msg) => {
|
||||
drawResults(input, msg.data, canvas);
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
});
|
||||
}
|
||||
timeStamp = performance.now();
|
||||
// const offscreen = image.transferControlToOffscreen();
|
||||
worker.postMessage({ image, config });
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
// perform detect if live video or not video at all
|
||||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
// perform detection
|
||||
webWorker(input, data, canvas);
|
||||
}
|
||||
}
|
||||
|
||||
function setupGUI() {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
settings.addBoolean('Pause', false, (val) => {
|
||||
const video = document.getElementById('video');
|
||||
const canvas = document.getElementById('canvas');
|
||||
if (val) video.pause();
|
||||
else video.play();
|
||||
runHumanDetect(video, canvas);
|
||||
});
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addBoolean('Draw Boxes', false);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
settings.addBoolean('Fill Polygons', true);
|
||||
settings.bindText('baseColor', ui.baseColor, config);
|
||||
settings.bindText('baseFont', ui.baseFont, config);
|
||||
settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config);
|
||||
settings.addHTML('line2', '<hr>'); settings.hideTitle('line2');
|
||||
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
|
||||
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
|
||||
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
|
||||
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
|
||||
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
|
||||
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
|
||||
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
|
||||
settings.addHTML('line3', '<hr>'); settings.hideTitle('line3');
|
||||
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
|
||||
config.face.detector.maxFaces = parseInt(val);
|
||||
config.body.maxDetections = parseInt(val);
|
||||
});
|
||||
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
|
||||
config.face.detector.skipFrames = parseInt(val);
|
||||
config.face.age.skipFrames = parseInt(val);
|
||||
config.hand.skipFrames = parseInt(val);
|
||||
});
|
||||
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
|
||||
config.face.detector.minConfidence = parseFloat(val);
|
||||
config.hand.minConfidence = parseFloat(val);
|
||||
});
|
||||
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
|
||||
config.face.detector.scoreThreshold = parseFloat(val);
|
||||
config.hand.scoreThreshold = parseFloat(val);
|
||||
config.body.scoreThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
|
||||
config.face.detector.iouThreshold = parseFloat(val);
|
||||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
}
|
||||
|
||||
async function setupCanvas(input) {
|
||||
// setup canvas object to same size as input as camera resolution may change
|
||||
const canvas = document.getElementById('canvas');
|
||||
canvas.width = input.width;
|
||||
canvas.height = input.height;
|
||||
return canvas;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupCamera() {
|
||||
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
|
||||
const video = document.getElementById('video');
|
||||
if (!navigator.mediaDevices) {
|
||||
document.getElementById('log').innerText = 'Video not supported';
|
||||
return null;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight },
|
||||
});
|
||||
video.srcObject = stream;
|
||||
return new Promise((resolve) => {
|
||||
video.onloadedmetadata = () => {
|
||||
video.width = video.videoWidth;
|
||||
video.height = video.videoHeight;
|
||||
video.play();
|
||||
resolve(video);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupImage() {
|
||||
const image = document.getElementById('image');
|
||||
image.width = window.innerWidth;
|
||||
image.height = window.innerHeight;
|
||||
return new Promise((resolve) => {
|
||||
image.onload = () => resolve(image);
|
||||
image.src = 'sample.jpg';
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// initialize tensorflow
|
||||
await human.tf.setBackend('webgl');
|
||||
await human.tf.ready();
|
||||
// setup ui control panel
|
||||
await setupGUI();
|
||||
// setup webcam
|
||||
const video = await setupCamera();
|
||||
// or setup image
|
||||
// const image = await setupImage();
|
||||
// setup output canvas from input object, select video or image
|
||||
const canvas = await setupCanvas(video);
|
||||
// run actual detection. if input is video, it will run in a loop else it will run only once
|
||||
runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
window.onload = main;
|
||||
window.onresize = main;
|
||||
|
|
@ -3887,21 +3887,20 @@ var require_ssrnet = __commonJS((exports2) => {
|
|||
let last = {age: 0, gender: ""};
|
||||
let frame = 0;
|
||||
async function getImage(image, size) {
|
||||
const tensor = tf2.tidy(() => {
|
||||
const buffer = tf2.browser.fromPixels(image);
|
||||
const resize = tf2.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf2.cast(tf2.expandDims(resize, 0), "float32");
|
||||
return expand;
|
||||
});
|
||||
return tensor;
|
||||
const buffer = tf2.browser.fromPixels(image);
|
||||
const resize = tf2.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf2.cast(tf2.expandDims(resize, 0), "float32");
|
||||
return expand;
|
||||
}
|
||||
async function loadAge(config) {
|
||||
if (!models2.age)
|
||||
models2.age = await tf2.loadGraphModel(config.face.age.modelPath);
|
||||
return models2.age;
|
||||
}
|
||||
async function loadGender(config) {
|
||||
if (!models2.gender)
|
||||
models2.gender = await tf2.loadGraphModel(config.face.gender.modelPath);
|
||||
return models2.gender;
|
||||
}
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
|
|
@ -3959,6 +3958,7 @@ var require_emotion = __commonJS((exports2) => {
|
|||
async function load(config) {
|
||||
if (!models2.emotion)
|
||||
models2.emotion = await tf2.loadGraphModel(config.face.emotion.modelPath);
|
||||
return models2.emotion;
|
||||
}
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
|
|
@ -5142,9 +5142,12 @@ const handpose = require_handpose();
|
|||
const defaults = require_config().default;
|
||||
const models = {
|
||||
facemesh: null,
|
||||
blazeface: null,
|
||||
ssrnet: null,
|
||||
iris: null
|
||||
posenet: null,
|
||||
handpose: null,
|
||||
iris: null,
|
||||
age: null,
|
||||
gender: null,
|
||||
emotion: null
|
||||
};
|
||||
function mergeDeep(...objects) {
|
||||
const isObject = (obj) => obj && typeof obj === "object";
|
||||
|
|
@ -5166,19 +5169,18 @@ function mergeDeep(...objects) {
|
|||
async function detect(input, userConfig) {
|
||||
return new Promise(async (resolve) => {
|
||||
const config = mergeDeep(defaults, userConfig);
|
||||
if (config.face.age.enabled)
|
||||
await ssrnet.loadAge(config);
|
||||
if (config.face.gender.enabled)
|
||||
await ssrnet.loadGender(config);
|
||||
if (config.face.emotion.enabled)
|
||||
await emotion.load(config);
|
||||
if (config.face.enabled && !models.facemesh)
|
||||
models.facemesh = await facemesh.load(config.face);
|
||||
if (config.body.enabled && !models.posenet)
|
||||
models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose)
|
||||
models.handpose = await handpose.load(config.hand);
|
||||
if (config.face.enabled && !models.facemesh)
|
||||
models.facemesh = await facemesh.load(config.face);
|
||||
tf.engine().startScope();
|
||||
if (config.face.enabled && config.face.age.enabled && !models.age)
|
||||
models.age = await ssrnet.loadAge(config);
|
||||
if (config.face.enabled && config.face.gender.enabled && !models.gender)
|
||||
models.gender = await ssrnet.loadGender(config);
|
||||
if (config.face.enabled && config.face.emotion.enabled && !models.emotion)
|
||||
models.emotion = await emotion.load(config);
|
||||
let savedWebglPackDepthwiseConvFlag;
|
||||
if (tf.getBackend() === "webgl") {
|
||||
savedWebglPackDepthwiseConvFlag = tf.env().get("WEBGL_PACK_DEPTHWISECONV");
|
||||
|
|
@ -5188,25 +5190,30 @@ async function detect(input, userConfig) {
|
|||
let timeStamp;
|
||||
timeStamp = performance.now();
|
||||
let poseRes = [];
|
||||
tf.engine().startScope();
|
||||
if (config.body.enabled)
|
||||
poseRes = await models.posenet.estimatePoses(input, config.body);
|
||||
tf.engine().endScope();
|
||||
perf.body = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
let handRes = [];
|
||||
tf.engine().startScope();
|
||||
if (config.hand.enabled)
|
||||
handRes = await models.handpose.estimateHands(input, config.hand);
|
||||
tf.engine().endScope();
|
||||
perf.hand = Math.trunc(performance.now() - timeStamp);
|
||||
const faceRes = [];
|
||||
if (config.face.enabled) {
|
||||
timeStamp = performance.now();
|
||||
tf.engine().startScope();
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(performance.now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
timeStamp = performance.now();
|
||||
const ssrdata = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
const ssrData = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
face.image.dispose();
|
||||
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
|
||||
|
|
@ -5215,15 +5222,15 @@ async function detect(input, userConfig) {
|
|||
box: face.box,
|
||||
mesh: face.mesh,
|
||||
annotations: face.annotations,
|
||||
age: ssrdata.age,
|
||||
gender: ssrdata.gender,
|
||||
emotion: emotiondata,
|
||||
age: ssrData.age,
|
||||
gender: ssrData.gender,
|
||||
emotion: emotionData,
|
||||
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
|
||||
});
|
||||
}
|
||||
tf.engine().endScope();
|
||||
}
|
||||
tf.env().set("WEBGL_PACK_DEPTHWISECONV", savedWebglPackDepthwiseConvFlag);
|
||||
tf.engine().endScope();
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf});
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -37,7 +37,7 @@
|
|||
"start": "node --trace-warnings --trace-uncaught --no-deprecation demo/demo-node.js",
|
||||
"lint": "eslint src/*.js demo/*.js",
|
||||
"build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --outfile=dist/human.js src/index.js",
|
||||
"build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --external:fs --outfile=dist/human.esm.js src/index.js",
|
||||
"build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --outfile=dist/human.esm.js src/index.js",
|
||||
"build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --outfile=dist/human.esm-nobundle.js src/index.js",
|
||||
"build-node-bundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --minify --outfile=dist/human.cjs src/index.js",
|
||||
"build-node-nobundle": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --outfile=dist/human-nobundle.cjs src/index.js",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ function getImage(image, size) {
|
|||
|
||||
async function load(config) {
|
||||
if (!models.emotion) models.emotion = await tf.loadGraphModel(config.face.emotion.modelPath);
|
||||
return models.emotion;
|
||||
}
|
||||
|
||||
async function predict(image, config) {
|
||||
|
|
@ -31,7 +32,7 @@ async function predict(image, config) {
|
|||
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
|
||||
const [r, g, b] = tf.split(resize, 3, 3);
|
||||
if (config.face.emotion.useGrayscale) {
|
||||
// 0.2989 * R + 0.5870 * G + 0.1140 * B // https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
const r1 = tf.mul(r, [0.2989]);
|
||||
const g1 = tf.mul(g, [0.5870]);
|
||||
const b1 = tf.mul(b, [0.1140]);
|
||||
|
|
|
|||
40
src/index.js
40
src/index.js
|
|
@ -6,13 +6,18 @@ const posenet = require('./posenet/posenet.js');
|
|||
const handpose = require('./handpose/handpose.js');
|
||||
const defaults = require('./config.js').default;
|
||||
|
||||
// object that contains all initialized models
|
||||
const models = {
|
||||
facemesh: null,
|
||||
blazeface: null,
|
||||
ssrnet: null,
|
||||
posenet: null,
|
||||
handpose: null,
|
||||
iris: null,
|
||||
age: null,
|
||||
gender: null,
|
||||
emotion: null,
|
||||
};
|
||||
|
||||
// helper function that performs deep merge of multiple objects so it allows full inheriance with overrides
|
||||
function mergeDeep(...objects) {
|
||||
const isObject = (obj) => obj && typeof obj === 'object';
|
||||
return objects.reduce((prev, obj) => {
|
||||
|
|
@ -37,15 +42,14 @@ async function detect(input, userConfig) {
|
|||
const config = mergeDeep(defaults, userConfig);
|
||||
|
||||
// load models if enabled
|
||||
if (config.face.age.enabled) await ssrnet.loadAge(config);
|
||||
if (config.face.gender.enabled) await ssrnet.loadGender(config);
|
||||
if (config.face.emotion.enabled) await emotion.load(config);
|
||||
if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face);
|
||||
if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand);
|
||||
if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face);
|
||||
|
||||
tf.engine().startScope();
|
||||
if (config.face.enabled && config.face.age.enabled && !models.age) models.age = await ssrnet.loadAge(config);
|
||||
if (config.face.enabled && config.face.gender.enabled && !models.gender) models.gender = await ssrnet.loadGender(config);
|
||||
if (config.face.enabled && config.face.emotion.enabled && !models.emotion) models.emotion = await emotion.load(config);
|
||||
|
||||
// explictly enable depthwiseconv since it's diasabled by default due to issues with large shaders
|
||||
let savedWebglPackDepthwiseConvFlag;
|
||||
if (tf.getBackend() === 'webgl') {
|
||||
savedWebglPackDepthwiseConvFlag = tf.env().get('WEBGL_PACK_DEPTHWISECONV');
|
||||
|
|
@ -58,29 +62,34 @@ async function detect(input, userConfig) {
|
|||
// run posenet
|
||||
timeStamp = performance.now();
|
||||
let poseRes = [];
|
||||
tf.engine().startScope();
|
||||
if (config.body.enabled) poseRes = await models.posenet.estimatePoses(input, config.body);
|
||||
tf.engine().endScope();
|
||||
perf.body = Math.trunc(performance.now() - timeStamp);
|
||||
|
||||
// run handpose
|
||||
timeStamp = performance.now();
|
||||
let handRes = [];
|
||||
tf.engine().startScope();
|
||||
if (config.hand.enabled) handRes = await models.handpose.estimateHands(input, config.hand);
|
||||
tf.engine().endScope();
|
||||
perf.hand = Math.trunc(performance.now() - timeStamp);
|
||||
|
||||
// run facemesh, includes blazeface and iris
|
||||
const faceRes = [];
|
||||
if (config.face.enabled) {
|
||||
timeStamp = performance.now();
|
||||
tf.engine().startScope();
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(performance.now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
// run ssr-net age & gender, inherits face from blazeface
|
||||
timeStamp = performance.now();
|
||||
const ssrdata = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
|
||||
const ssrData = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
// run emotion, inherits face from blazeface
|
||||
timeStamp = performance.now();
|
||||
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
face.image.dispose();
|
||||
// calculate iris distance
|
||||
|
|
@ -93,18 +102,19 @@ async function detect(input, userConfig) {
|
|||
box: face.box,
|
||||
mesh: face.mesh,
|
||||
annotations: face.annotations,
|
||||
age: ssrdata.age,
|
||||
gender: ssrdata.gender,
|
||||
emotion: emotiondata,
|
||||
age: ssrData.age,
|
||||
gender: ssrData.gender,
|
||||
emotion: emotionData,
|
||||
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
|
||||
});
|
||||
}
|
||||
tf.engine().endScope();
|
||||
}
|
||||
|
||||
// set depthwiseconv to original value
|
||||
tf.env().set('WEBGL_PACK_DEPTHWISECONV', savedWebglPackDepthwiseConvFlag);
|
||||
|
||||
tf.engine().endScope();
|
||||
// combine results
|
||||
// combine and return results
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,22 +5,20 @@ let last = { age: 0, gender: '' };
|
|||
let frame = 0;
|
||||
|
||||
async function getImage(image, size) {
|
||||
const tensor = tf.tidy(() => {
|
||||
const buffer = tf.browser.fromPixels(image);
|
||||
const resize = tf.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf.cast(tf.expandDims(resize, 0), 'float32');
|
||||
// const normalize = tf.mul(expand, [1.0 / 1.0]);
|
||||
return expand;
|
||||
});
|
||||
return tensor;
|
||||
const buffer = tf.browser.fromPixels(image);
|
||||
const resize = tf.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf.cast(tf.expandDims(resize, 0), 'float32');
|
||||
return expand;
|
||||
}
|
||||
|
||||
async function loadAge(config) {
|
||||
if (!models.age) models.age = await tf.loadGraphModel(config.face.age.modelPath);
|
||||
return models.age;
|
||||
}
|
||||
|
||||
async function loadGender(config) {
|
||||
if (!models.gender) models.gender = await tf.loadGraphModel(config.face.gender.modelPath);
|
||||
return models.gender;
|
||||
}
|
||||
|
||||
async function predict(image, config) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue