mirror of https://github.com/vladmandic/human
added error handling
Vladimir Mandic
parent
4f160ba86f
commit
5a1b34eaad
|
|
@ -7,6 +7,7 @@ const ui = {
|
|||
baseLabel: 'rgba(255, 200, 255, 0.8)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
baseLineWidth: 16,
|
||||
busy: false,
|
||||
};
|
||||
|
||||
const config = {
|
||||
|
|
@ -16,13 +17,13 @@ const config = {
|
|||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
iris: { enabled: false },
|
||||
age: { enabled: true, skipFrames: 10 },
|
||||
gender: { enabled: true },
|
||||
emotion: { enabled: true, minConfidence: 0.5, useGrayscale: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
body: { enabled: false, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
hand: { enabled: false, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
};
|
||||
let settings;
|
||||
let worker;
|
||||
|
|
@ -245,10 +246,16 @@ function webWorker(input, image, canvas) {
|
|||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
timeStamp = performance.now();
|
||||
// perform detect if live video or not video at all
|
||||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
if (input.srcObject) {
|
||||
// if video not ready, just redo
|
||||
const live = (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
|
||||
if (!live) {
|
||||
if (!input.paused) log(`Video not ready: state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
|
||||
setTimeout(() => runHumanDetect(input, canvas), 500);
|
||||
return;
|
||||
}
|
||||
if (settings.getValue('Use Web Worker')) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
|
|
@ -265,7 +272,8 @@ async function runHumanDetect(input, canvas) {
|
|||
} catch (err) {
|
||||
log('Error during execution:', err.message);
|
||||
}
|
||||
drawResults(input, result, canvas);
|
||||
if (result.error) log(result.error);
|
||||
else drawResults(input, result, canvas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -333,7 +341,7 @@ function setupUI() {
|
|||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
|
||||
settings.addBoolean('Use Web Worker', true);
|
||||
settings.addBoolean('Use Web Worker', false);
|
||||
settings.addBoolean('Draw Boxes', true);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
|
|
@ -342,21 +350,20 @@ function setupUI() {
|
|||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
}
|
||||
|
||||
async function setupCanvas(input) {
|
||||
// setup canvas object to same size as input as camera resolution may change
|
||||
const canvas = document.getElementById('canvas');
|
||||
canvas.width = input.width;
|
||||
canvas.height = input.height;
|
||||
return canvas;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupCamera() {
|
||||
log('Setting up camera');
|
||||
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
|
||||
if (ui.busy) return null;
|
||||
ui.busy = true;
|
||||
const video = document.getElementById('video');
|
||||
const canvas = document.getElementById('canvas');
|
||||
const output = document.getElementById('log');
|
||||
const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false;
|
||||
log('Setting up camera: live:', live);
|
||||
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
|
||||
if (!navigator.mediaDevices) {
|
||||
document.getElementById('log').innerText = 'Video not supported';
|
||||
const msg = 'Camera access not supported';
|
||||
output.innerText = msg;
|
||||
log(msg);
|
||||
return null;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
|
|
@ -365,11 +372,15 @@ async function setupCamera() {
|
|||
});
|
||||
video.srcObject = stream;
|
||||
return new Promise((resolve) => {
|
||||
video.onloadedmetadata = () => {
|
||||
video.onloadeddata = async () => {
|
||||
video.width = video.videoWidth;
|
||||
video.height = video.videoHeight;
|
||||
video.play();
|
||||
video.pause();
|
||||
canvas.width = video.videoWidth;
|
||||
canvas.height = video.videoHeight;
|
||||
if (live) video.play();
|
||||
ui.busy = false;
|
||||
// do once more because onresize events can be delayed or skipped
|
||||
if (video.width !== window.innerWidth) await setupCamera();
|
||||
resolve(video);
|
||||
};
|
||||
});
|
||||
|
|
@ -387,16 +398,15 @@ async function setupImage() {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
log('Human starting ...');
|
||||
log('Human demo starting ...');
|
||||
|
||||
// setup ui control panel
|
||||
await setupUI();
|
||||
// setup webcam
|
||||
const input = await setupCamera();
|
||||
await setupCamera();
|
||||
// or setup image
|
||||
// const input = await setupImage();
|
||||
// setup output canvas from input object
|
||||
await setupCanvas(input);
|
||||
|
||||
const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`;
|
||||
document.getElementById('log').innerText = msg;
|
||||
|
|
@ -407,4 +417,4 @@ async function main() {
|
|||
}
|
||||
|
||||
window.onload = main;
|
||||
window.onresize = main;
|
||||
window.onresize = setupCamera;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
<head>
|
||||
<script src="https://cdn.jsdelivr.net/npm/quicksettings@latest/quicksettings.min.js"></script>
|
||||
<script src="../dist/human.js"></script>
|
||||
<script src="./demo-iife.js"></script>
|
||||
</head>
|
||||
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'">
|
||||
<div id="main">
|
||||
|
|
@ -10,4 +9,414 @@
|
|||
<canvas id="canvas"></canvas>
|
||||
<div id="log">Starting Human library</div>
|
||||
</div>
|
||||
<script>
|
||||
/* global QuickSettings */
|
||||
|
||||
const ui = {
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseLabel: 'rgba(255, 200, 255, 0.8)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
baseLineWidth: 16,
|
||||
};
|
||||
|
||||
const config = {
|
||||
backend: 'webgl',
|
||||
console: true,
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 10 },
|
||||
gender: { enabled: true },
|
||||
emotion: { enabled: true, minConfidence: 0.5, useGrayscale: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
};
|
||||
let settings;
|
||||
let worker;
|
||||
let timeStamp;
|
||||
const fps = [];
|
||||
|
||||
function str(...msg) {
|
||||
if (!Array.isArray(msg)) return msg;
|
||||
let line = '';
|
||||
for (const entry of msg) {
|
||||
if (typeof entry === 'object') line += JSON.stringify(entry).replace(/{|}|"|\[|\]/g, '').replace(/,/g, ', ');
|
||||
else line += entry;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
const log = (...msg) => {
|
||||
// eslint-disable-next-line no-console
|
||||
if (config.console) console.log(...msg);
|
||||
};
|
||||
|
||||
async function drawFace(result, canvas) {
|
||||
if (!result) return;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
for (const face of result) {
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
|
||||
}
|
||||
const labelAgeGender = `${face.gender || ''} ${face.age || ''}`;
|
||||
const labelIris = face.iris ? `iris: ${face.iris}` : '';
|
||||
const labelEmotion = face.emotion && face.emotion[0] ? `emotion: ${Math.trunc(100 * face.emotion[0].score)}% ${face.emotion[0].emotion}` : '';
|
||||
ctx.fillStyle = ui.baseLabel;
|
||||
ctx.fillText(`${Math.trunc(100 * face.confidence)}% face ${labelAgeGender} ${labelIris} ${labelEmotion}`, face.box[0] + 2, face.box[1] + 22);
|
||||
ctx.stroke();
|
||||
ctx.lineWidth = 1;
|
||||
if (face.mesh) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of face.mesh) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
for (let i = 0; i < human.facemesh.triangulation.length / 3; i++) {
|
||||
const points = [
|
||||
human.facemesh.triangulation[i * 3 + 0],
|
||||
human.facemesh.triangulation[i * 3 + 1],
|
||||
human.facemesh.triangulation[i * 3 + 2],
|
||||
].map((index) => face.mesh[index]);
|
||||
const path = new Path2D();
|
||||
path.moveTo(points[0][0], points[0][1]);
|
||||
for (const point of points) {
|
||||
path.lineTo(point[0], point[1]);
|
||||
}
|
||||
path.closePath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.stroke(path);
|
||||
if (settings.getValue('Fill Polygons')) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.fill(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawBody(result, canvas) {
|
||||
if (!result) return;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
for (const pose of result) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of pose.keypoints) {
|
||||
ctx.beginPath();
|
||||
ctx.arc(point.position.x, point.position.y, 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const path = new Path2D();
|
||||
let part;
|
||||
// torso
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// legs
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// draw all
|
||||
ctx.stroke(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawHand(result, canvas) {
|
||||
if (!result) return;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
window.result = result;
|
||||
for (const hand of result) {
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
|
||||
ctx.fillStyle = ui.baseLabel;
|
||||
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
|
||||
ctx.stroke();
|
||||
}
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of hand.landmarks) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const addPart = (part) => {
|
||||
for (let i = 1; i < part.length; i++) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`;
|
||||
ctx.moveTo(part[i - 1][0], part[i - 1][1]);
|
||||
ctx.lineTo(part[i][0], part[i][1]);
|
||||
ctx.stroke();
|
||||
}
|
||||
};
|
||||
addPart(hand.annotations.indexFinger);
|
||||
addPart(hand.annotations.middleFinger);
|
||||
addPart(hand.annotations.ringFinger);
|
||||
addPart(hand.annotations.pinky);
|
||||
addPart(hand.annotations.thumb);
|
||||
addPart(hand.annotations.palmBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawResults(input, result, canvas) {
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
|
||||
fps.push(1000 / (performance.now() - timeStamp));
|
||||
if (fps.length > 20) fps.shift();
|
||||
settings.setValue('FPS', Math.round(10 * fps.reduce((a, b) => a + b) / fps.length) / 10);
|
||||
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
|
||||
// draw image from video
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
// draw all results
|
||||
drawFace(result.face, canvas);
|
||||
drawBody(result.body, canvas);
|
||||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
|
||||
const gpu = engine.backendInstance ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
|
||||
document.getElementById('log').innerText = `
|
||||
TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu}
|
||||
Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
|
||||
`;
|
||||
}
|
||||
|
||||
// simple wrapper for worker.postmessage that creates worker if one does not exist
|
||||
function webWorker(input, image, canvas) {
|
||||
if (!worker) {
|
||||
// create new webworker and add event handler only once
|
||||
log('Creating worker thread');
|
||||
worker = new Worker('demo-esm-webworker.js', { type: 'module' });
|
||||
// after receiving message from webworker, parse&draw results and send new frame for processing
|
||||
worker.addEventListener('message', (msg) => drawResults(input, msg.data, canvas));
|
||||
}
|
||||
// pass image data as arraybuffer to worker by reference to avoid copy
|
||||
worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]);
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
timeStamp = performance.now();
|
||||
// perform detect if live video or not video at all
|
||||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
if (settings.getValue('Use Web Worker')) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
// perform detection in worker
|
||||
webWorker(input, data, canvas);
|
||||
} else {
|
||||
let result = {};
|
||||
try {
|
||||
// perform detection
|
||||
result = await human.detect(input, config);
|
||||
} catch (err) {
|
||||
log('Error during execution:', err.message);
|
||||
}
|
||||
drawResults(input, result, canvas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function setupUI() {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
const style = document.createElement('style');
|
||||
// style.type = 'text/css';
|
||||
style.innerHTML = `
|
||||
.qs_main { font: 1rem "Segoe UI"; }
|
||||
.qs_label { font: 0.8rem "Segoe UI"; }
|
||||
.qs_title_bar { display: none; }
|
||||
.qs_content { background: darkslategray; }
|
||||
.qs_container { background: transparent; color: white; margin: 6px; padding: 6px; }
|
||||
.qs_checkbox_label { top: 2px; }
|
||||
.qs_button { width: -webkit-fill-available; font: 1rem "Segoe UI"; cursor: pointer; }
|
||||
`;
|
||||
document.getElementsByTagName('head')[0].appendChild(style);
|
||||
settings.addButton('Play/Pause', () => {
|
||||
const video = document.getElementById('video');
|
||||
const canvas = document.getElementById('canvas');
|
||||
if (!video.paused) {
|
||||
document.getElementById('log').innerText = 'Paused ...';
|
||||
video.pause();
|
||||
} else {
|
||||
document.getElementById('log').innerText = 'Starting Human Library ...';
|
||||
video.play();
|
||||
}
|
||||
runHumanDetect(video, canvas);
|
||||
});
|
||||
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => config.backend = val.value);
|
||||
settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title');
|
||||
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
|
||||
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
|
||||
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
|
||||
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
|
||||
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
|
||||
settings.addBoolean('Face Emotion', config.face.emotion.enabled, (val) => config.face.emotion.enabled = val);
|
||||
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
|
||||
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
|
||||
settings.addHTML('title', 'Model Parameters'); settings.hideTitle('title');
|
||||
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
|
||||
config.face.detector.maxFaces = parseInt(val);
|
||||
config.body.maxDetections = parseInt(val);
|
||||
});
|
||||
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
|
||||
config.face.detector.skipFrames = parseInt(val);
|
||||
config.face.emotion.skipFrames = parseInt(val);
|
||||
config.face.age.skipFrames = parseInt(val);
|
||||
config.hand.skipFrames = parseInt(val);
|
||||
});
|
||||
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
|
||||
config.face.detector.minConfidence = parseFloat(val);
|
||||
config.face.emotion.minConfidence = parseFloat(val);
|
||||
config.hand.minConfidence = parseFloat(val);
|
||||
});
|
||||
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
|
||||
config.face.detector.scoreThreshold = parseFloat(val);
|
||||
config.hand.scoreThreshold = parseFloat(val);
|
||||
config.body.scoreThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
|
||||
config.face.detector.iouThreshold = parseFloat(val);
|
||||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
|
||||
settings.addBoolean('Use Web Worker', false);
|
||||
settings.addBoolean('Draw Boxes', true);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
settings.addBoolean('Fill Polygons', true);
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
}
|
||||
|
||||
async function setupCanvas(input) {
|
||||
// setup canvas object to same size as input as camera resolution may change
|
||||
const canvas = document.getElementById('canvas');
|
||||
canvas.width = input.width;
|
||||
canvas.height = input.height;
|
||||
return canvas;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupCamera() {
|
||||
log('Setting up camera');
|
||||
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
|
||||
const video = document.getElementById('video');
|
||||
if (!navigator.mediaDevices) {
|
||||
document.getElementById('log').innerText = 'Video not supported';
|
||||
return null;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight },
|
||||
});
|
||||
video.srcObject = stream;
|
||||
return new Promise((resolve) => {
|
||||
video.onloadedmetadata = () => {
|
||||
video.width = video.videoWidth;
|
||||
video.height = video.videoHeight;
|
||||
video.play();
|
||||
video.pause();
|
||||
resolve(video);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupImage() {
|
||||
const image = document.getElementById('image');
|
||||
image.width = window.innerWidth;
|
||||
image.height = window.innerHeight;
|
||||
return new Promise((resolve) => {
|
||||
image.onload = () => resolve(image);
|
||||
image.src = 'sample.jpg';
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log('Human starting ...');
|
||||
|
||||
// setup ui control panel
|
||||
await setupUI();
|
||||
// setup webcam
|
||||
const input = await setupCamera();
|
||||
// or setup image
|
||||
// const input = await setupImage();
|
||||
// setup output canvas from input object
|
||||
await setupCanvas(input);
|
||||
|
||||
const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`;
|
||||
document.getElementById('log').innerText = msg;
|
||||
log(msg);
|
||||
|
||||
// run actual detection. if input is video, it will run in a loop else it will run only once
|
||||
// runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
window.onload = main;
|
||||
window.onresize = main;
|
||||
</script>
|
||||
</body>
|
||||
|
|
|
|||
|
|
@ -1,317 +0,0 @@
|
|||
/* global human, QuickSettings */
|
||||
|
||||
const ui = {
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
baseLineWidth: 16,
|
||||
};
|
||||
|
||||
const config = {
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 5 },
|
||||
gender: { enabled: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
};
|
||||
let settings;
|
||||
|
||||
async function drawFace(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
for (const face of result) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
|
||||
}
|
||||
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
|
||||
ctx.stroke();
|
||||
ctx.lineWidth = 1;
|
||||
if (face.mesh) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of face.mesh) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
for (let i = 0; i < human.facemesh.triangulation.length / 3; i++) {
|
||||
const points = [
|
||||
human.facemesh.triangulation[i * 3 + 0],
|
||||
human.facemesh.triangulation[i * 3 + 1],
|
||||
human.facemesh.triangulation[i * 3 + 2],
|
||||
].map((index) => face.mesh[index]);
|
||||
const path = new Path2D();
|
||||
path.moveTo(points[0][0], points[0][1]);
|
||||
for (const point of points) {
|
||||
path.lineTo(point[0], point[1]);
|
||||
}
|
||||
path.closePath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.stroke(path);
|
||||
if (settings.getValue('Fill Polygons')) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
|
||||
ctx.fill(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawBody(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
for (const pose of result) {
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of pose.keypoints) {
|
||||
ctx.beginPath();
|
||||
ctx.arc(point.position.x, point.position.y, 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const path = new Path2D();
|
||||
let part;
|
||||
// torso
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// legs
|
||||
part = pose.keypoints.find((a) => a.part === 'leftHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightHip');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightKnee');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightAnkle');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'leftShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'leftWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// arms
|
||||
part = pose.keypoints.find((a) => a.part === 'rightShoulder');
|
||||
path.moveTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightElbow');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
part = pose.keypoints.find((a) => a.part === 'rightWrist');
|
||||
path.lineTo(part.position.x, part.position.y);
|
||||
// draw all
|
||||
ctx.stroke(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function drawHand(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.font = ui.baseFont;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
window.result = result;
|
||||
for (const hand of result) {
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
|
||||
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
|
||||
ctx.stroke();
|
||||
}
|
||||
if (settings.getValue('Draw Points')) {
|
||||
for (const point of hand.landmarks) {
|
||||
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (settings.getValue('Draw Polygons')) {
|
||||
const addPart = (part) => {
|
||||
for (let i = 1; i < part.length; i++) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`;
|
||||
ctx.moveTo(part[i - 1][0], part[i - 1][1]);
|
||||
ctx.lineTo(part[i][0], part[i][1]);
|
||||
ctx.stroke();
|
||||
}
|
||||
};
|
||||
addPart(hand.annotations.indexFinger);
|
||||
addPart(hand.annotations.middleFinger);
|
||||
addPart(hand.annotations.ringFinger);
|
||||
addPart(hand.annotations.pinky);
|
||||
addPart(hand.annotations.thumb);
|
||||
addPart(hand.annotations.palmBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
const log = document.getElementById('log');
|
||||
const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
|
||||
// perform detect if live video or not video at all
|
||||
if (live || !(input instanceof HTMLVideoElement)) {
|
||||
// perform detection
|
||||
const t0 = performance.now();
|
||||
const result = await human.detect(input, config);
|
||||
if (!result) return;
|
||||
const t1 = performance.now();
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (t1 - t0)));
|
||||
// draw image from video
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
// draw all results
|
||||
drawFace(result.face, canvas);
|
||||
drawBody(result.body, canvas);
|
||||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
log.innerText = `
|
||||
TFJS Version: ${human.tf.version_core} Memory: ${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors
|
||||
GPU Memory: used ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes free ${Math.floor(1024 * 1024 * engine.backendInstance.numMBBeforeWarning).toLocaleString()} bytes
|
||||
Result Object Size: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes
|
||||
`;
|
||||
// rinse & repeate
|
||||
// if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes
|
||||
if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
|
||||
}
|
||||
}
|
||||
|
||||
function setupGUI() {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
settings.addBoolean('Pause', false, (val) => {
|
||||
const video = document.getElementById('video');
|
||||
const canvas = document.getElementById('canvas');
|
||||
if (val) video.pause();
|
||||
else video.play();
|
||||
runHumanDetect(video, canvas);
|
||||
});
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addBoolean('Draw Boxes', false);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
settings.addBoolean('Fill Polygons', true);
|
||||
settings.bindText('baseColor', ui.baseColor, config);
|
||||
settings.bindText('baseFont', ui.baseFont, config);
|
||||
settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config);
|
||||
settings.addHTML('line2', '<hr>'); settings.hideTitle('line2');
|
||||
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
|
||||
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
|
||||
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
|
||||
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
|
||||
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
|
||||
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
|
||||
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
|
||||
settings.addHTML('line3', '<hr>'); settings.hideTitle('line3');
|
||||
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
|
||||
config.face.detector.maxFaces = parseInt(val);
|
||||
config.body.maxDetections = parseInt(val);
|
||||
});
|
||||
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
|
||||
config.face.detector.skipFrames = parseInt(val);
|
||||
config.face.age.skipFrames = parseInt(val);
|
||||
config.hand.skipFrames = parseInt(val);
|
||||
});
|
||||
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
|
||||
config.face.detector.minConfidence = parseFloat(val);
|
||||
config.hand.minConfidence = parseFloat(val);
|
||||
});
|
||||
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
|
||||
config.face.detector.scoreThreshold = parseFloat(val);
|
||||
config.hand.scoreThreshold = parseFloat(val);
|
||||
config.body.scoreThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
|
||||
config.face.detector.iouThreshold = parseFloat(val);
|
||||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
}
|
||||
|
||||
async function setupCanvas(input) {
|
||||
// setup canvas object to same size as input as camera resolution may change
|
||||
const canvas = document.getElementById('canvas');
|
||||
canvas.width = input.width;
|
||||
canvas.height = input.height;
|
||||
return canvas;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupCamera() {
|
||||
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
|
||||
const video = document.getElementById('video');
|
||||
if (!navigator.mediaDevices) {
|
||||
document.getElementById('log').innerText = 'Video not supported';
|
||||
return null;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: false,
|
||||
video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight },
|
||||
});
|
||||
video.srcObject = stream;
|
||||
return new Promise((resolve) => {
|
||||
video.onloadedmetadata = () => {
|
||||
video.width = video.videoWidth;
|
||||
video.height = video.videoHeight;
|
||||
video.play();
|
||||
resolve(video);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
async function setupImage() {
|
||||
const image = document.getElementById('image');
|
||||
image.width = window.innerWidth;
|
||||
image.height = window.innerHeight;
|
||||
return new Promise((resolve) => {
|
||||
image.onload = () => resolve(image);
|
||||
image.src = 'sample.jpg';
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// initialize tensorflow
|
||||
await human.tf.setBackend('webgl');
|
||||
await human.tf.ready();
|
||||
// setup ui control panel
|
||||
await setupGUI();
|
||||
// setup webcam
|
||||
const video = await setupCamera();
|
||||
// or setup image
|
||||
// const image = await setupImage();
|
||||
// setup output canvas from input object, select video or image
|
||||
const canvas = await setupCanvas(video);
|
||||
// run actual detection. if input is video, it will run in a loop else it will run only once
|
||||
runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
window.onload = main;
|
||||
window.onresize = main;
|
||||
|
|
@ -2,7 +2,7 @@ const tf = require('@tensorflow/tfjs-node');
|
|||
const fs = require('fs');
|
||||
const process = require('process');
|
||||
const console = require('console');
|
||||
const human = require('..'); // this would be '@vladmandic/human'
|
||||
const human = require('..'); // this resolves to project root which is '@vladmandic/human'
|
||||
|
||||
const logger = new console.Console({
|
||||
stdout: process.stdout,
|
||||
|
|
@ -24,6 +24,8 @@ const logger = new console.Console({
|
|||
});
|
||||
|
||||
const config = {
|
||||
backend: 'tensorflow',
|
||||
console: true,
|
||||
face: {
|
||||
enabled: false,
|
||||
detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ var require_blazeface = __commonJS((exports2) => {
|
|||
this.scoreThreshold = config2.detector.scoreThreshold;
|
||||
}
|
||||
async getBoundingBoxes(inputImage) {
|
||||
if (!inputImage || inputImage.isDisposedInternal || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return null;
|
||||
const [detectedOutputs, boxes, scores] = tf2.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.width, this.height]);
|
||||
const normalizedImage = tf2.mul(tf2.sub(resizedImage.div(255), 0.5), 2);
|
||||
|
|
@ -5047,6 +5049,8 @@ var require_config = __commonJS((exports2) => {
|
|||
default: () => config_default
|
||||
});
|
||||
var config_default = {
|
||||
backend: "webgl",
|
||||
console: true,
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: {
|
||||
|
|
@ -5122,10 +5126,10 @@ var require_config = __commonJS((exports2) => {
|
|||
var require_package = __commonJS((exports2, module2) => {
|
||||
module2.exports = {
|
||||
name: "@vladmandic/human",
|
||||
version: "0.3.2",
|
||||
version: "0.3.3",
|
||||
description: "human: 3D Face Detection, Iris Tracking and Age & Gender Prediction",
|
||||
sideEffects: false,
|
||||
main: "dist/human.cjs",
|
||||
main: "dist/human-nobundle.cjs",
|
||||
module: "dist/human.esm.js",
|
||||
browser: "dist/human.esm.js",
|
||||
author: "Vladimir Mandic <mandic00@live.com>",
|
||||
|
|
@ -5200,6 +5204,11 @@ const models = {
|
|||
gender: null,
|
||||
emotion: null
|
||||
};
|
||||
const now = () => {
|
||||
if (typeof performance !== "undefined")
|
||||
return performance.now();
|
||||
return parseInt(Number(process.hrtime.bigint()) / 1e3 / 1e3);
|
||||
};
|
||||
const log = (...msg) => {
|
||||
if (config.console)
|
||||
console.log(...msg);
|
||||
|
|
@ -5221,9 +5230,29 @@ function mergeDeep(...objects) {
|
|||
return prev;
|
||||
}, {});
|
||||
}
|
||||
function sanity(input) {
|
||||
if (!input)
|
||||
return "input is not defined";
|
||||
const width = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
|
||||
if (!width || width === 0)
|
||||
return "input is empty";
|
||||
if (input.readyState && input.readyState <= 2)
|
||||
return "input is not ready";
|
||||
try {
|
||||
tf.getBackend();
|
||||
} catch {
|
||||
return "backend not loaded";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async function detect(input, userConfig) {
|
||||
return new Promise(async (resolve) => {
|
||||
config = mergeDeep(defaults, userConfig);
|
||||
const error = sanity(input);
|
||||
if (error) {
|
||||
log(error, input);
|
||||
return {error};
|
||||
}
|
||||
return new Promise(async (resolve) => {
|
||||
const loadedModels = Object.values(models).filter((a) => a).length;
|
||||
if (loadedModels === 0)
|
||||
log("Human library starting");
|
||||
|
|
@ -5251,29 +5280,33 @@ async function detect(input, userConfig) {
|
|||
models.emotion = await emotion.load(config);
|
||||
const perf = {};
|
||||
let timeStamp;
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const poseRes = config.body.enabled ? await models.posenet.estimatePoses(input, config.body) : [];
|
||||
tf.engine().endScope();
|
||||
perf.body = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
perf.body = Math.trunc(now() - timeStamp);
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const handRes = config.hand.enabled ? await models.handpose.estimateHands(input, config.hand) : [];
|
||||
tf.engine().endScope();
|
||||
perf.hand = Math.trunc(performance.now() - timeStamp);
|
||||
perf.hand = Math.trunc(now() - timeStamp);
|
||||
const faceRes = [];
|
||||
if (config.face.enabled) {
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(performance.now() - timeStamp);
|
||||
perf.face = Math.trunc(now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
timeStamp = performance.now();
|
||||
if (!face.image || face.image.isDisposedInternal) {
|
||||
log("face object is disposed:", face.image);
|
||||
continue;
|
||||
}
|
||||
timeStamp = now();
|
||||
const ssrData = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
perf.agegender = Math.trunc(now() - timeStamp);
|
||||
timeStamp = now();
|
||||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
perf.emotion = Math.trunc(now() - timeStamp);
|
||||
face.image.dispose();
|
||||
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
|
||||
faceRes.push({
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -3,7 +3,7 @@
|
|||
"version": "0.3.3",
|
||||
"description": "human: 3D Face Detection, Iris Tracking and Age & Gender Prediction",
|
||||
"sideEffects": false,
|
||||
"main": "dist/human.cjs",
|
||||
"main": "dist/human-nobundle.cjs",
|
||||
"module": "dist/human.esm.js",
|
||||
"browser": "dist/human.esm.js",
|
||||
"author": "Vladimir Mandic <mandic00@live.com>",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
export default {
|
||||
backend: 'webgl',
|
||||
console: true,
|
||||
face: {
|
||||
enabled: true, // refers to detector, but since all other face modules rely on detector, it should be a global
|
||||
detector: {
|
||||
|
|
|
|||
|
|
@ -83,6 +83,8 @@ class BlazeFaceModel {
|
|||
}
|
||||
|
||||
async getBoundingBoxes(inputImage) {
|
||||
// sanity check on input
|
||||
if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null;
|
||||
const [detectedOutputs, boxes, scores] = tf.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.width, this.height]);
|
||||
const normalizedImage = tf.mul(tf.sub(resizedImage.div(255), 0.5), 2);
|
||||
|
|
@ -101,7 +103,6 @@ class BlazeFaceModel {
|
|||
const decodedBounds = decodeBounds(prediction, this.anchors, this.inputSize);
|
||||
const logits = tf.slice(prediction, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf.sigmoid(logits).squeeze();
|
||||
// console.log(prediction, decodedBounds, logits, scoresOut);
|
||||
return [prediction, decodedBounds, scoresOut];
|
||||
});
|
||||
|
||||
|
|
|
|||
53
src/index.js
53
src/index.js
|
|
@ -19,6 +19,10 @@ const models = {
|
|||
gender: null,
|
||||
emotion: null,
|
||||
};
|
||||
const now = () => {
|
||||
if (typeof performance !== 'undefined') return performance.now();
|
||||
return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000);
|
||||
};
|
||||
|
||||
const log = (...msg) => {
|
||||
// eslint-disable-next-line no-console
|
||||
|
|
@ -44,11 +48,31 @@ function mergeDeep(...objects) {
|
|||
}, {});
|
||||
}
|
||||
|
||||
function sanity(input) {
|
||||
if (!input) return 'input is not defined';
|
||||
const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
|
||||
if (!width || (width === 0)) return 'input is empty';
|
||||
if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
|
||||
try {
|
||||
tf.getBackend();
|
||||
} catch {
|
||||
return 'backend not loaded';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function detect(input, userConfig) {
|
||||
// eslint-disable-next-line no-async-promise-executor
|
||||
return new Promise(async (resolve) => {
|
||||
config = mergeDeep(defaults, userConfig);
|
||||
|
||||
// sanity checks
|
||||
const error = sanity(input);
|
||||
if (error) {
|
||||
log(error, input);
|
||||
return { error };
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-async-promise-executor
|
||||
return new Promise(async (resolve) => {
|
||||
// check number of loaded models
|
||||
const loadedModels = Object.values(models).filter((a) => a).length;
|
||||
if (loadedModels === 0) log('Human library starting');
|
||||
|
|
@ -78,35 +102,40 @@ async function detect(input, userConfig) {
|
|||
let timeStamp;
|
||||
|
||||
// run posenet
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const poseRes = config.body.enabled ? await models.posenet.estimatePoses(input, config.body) : [];
|
||||
tf.engine().endScope();
|
||||
perf.body = Math.trunc(performance.now() - timeStamp);
|
||||
perf.body = Math.trunc(now() - timeStamp);
|
||||
|
||||
// run handpose
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const handRes = config.hand.enabled ? await models.handpose.estimateHands(input, config.hand) : [];
|
||||
tf.engine().endScope();
|
||||
perf.hand = Math.trunc(performance.now() - timeStamp);
|
||||
perf.hand = Math.trunc(now() - timeStamp);
|
||||
|
||||
// run facemesh, includes blazeface and iris
|
||||
const faceRes = [];
|
||||
if (config.face.enabled) {
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
tf.engine().startScope();
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(performance.now() - timeStamp);
|
||||
perf.face = Math.trunc(now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
// is something went wrong, skip the face
|
||||
if (!face.image || face.image.isDisposedInternal) {
|
||||
log('face object is disposed:', face.image);
|
||||
continue;
|
||||
}
|
||||
// run ssr-net age & gender, inherits face from blazeface
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
const ssrData = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
perf.agegender = Math.trunc(now() - timeStamp);
|
||||
// run emotion, inherits face from blazeface
|
||||
timeStamp = performance.now();
|
||||
timeStamp = now();
|
||||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
perf.emotion = Math.trunc(now() - timeStamp);
|
||||
face.image.dispose();
|
||||
// calculate iris distance
|
||||
// iris: array[ bottom, left, top, right, center ]
|
||||
|
|
|
|||
Loading…
Reference in New Issue