diff --git a/demo/demo-esm.js b/demo/demo-esm.js index b334913a..93899ee0 100644 --- a/demo/demo-esm.js +++ b/demo/demo-esm.js @@ -7,6 +7,7 @@ const ui = { baseLabel: 'rgba(255, 200, 255, 0.8)', baseFont: 'small-caps 1.2rem "Segoe UI"', baseLineWidth: 16, + busy: false, }; const config = { @@ -16,13 +17,13 @@ const config = { enabled: true, detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, mesh: { enabled: true }, - iris: { enabled: true }, + iris: { enabled: false }, age: { enabled: true, skipFrames: 10 }, gender: { enabled: true }, emotion: { enabled: true, minConfidence: 0.5, useGrayscale: true }, }, - body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, - hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, + body: { enabled: false, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, + hand: { enabled: false, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, }; let settings; let worker; @@ -245,10 +246,16 @@ function webWorker(input, image, canvas) { } async function runHumanDetect(input, canvas) { - const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false; timeStamp = performance.now(); // perform detect if live video or not video at all - if (live || !(input instanceof HTMLVideoElement)) { + if (input.srcObject) { + // if video not ready, just redo + const live = (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused); + if (!live) { + if (!input.paused) log(`Video not ready: state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`); + setTimeout(() => runHumanDetect(input, canvas), 500); + return; + } if (settings.getValue('Use Web Worker')) { // get image data from video as we cannot send html objects to webworker const offscreen = new OffscreenCanvas(canvas.width, canvas.height); @@ -265,7 +272,8 @@ async function runHumanDetect(input, canvas) { } catch (err) { log('Error during execution:', err.message); } - drawResults(input, result, canvas); + if (result.error) log(result.error); + else drawResults(input, result, canvas); } } } @@ -333,7 +341,7 @@ function setupUI() { config.hand.iouThreshold = parseFloat(val); }); settings.addHTML('title', 'UI Options'); settings.hideTitle('title'); - settings.addBoolean('Use Web Worker', true); + settings.addBoolean('Use Web Worker', false); settings.addBoolean('Draw Boxes', true); settings.addBoolean('Draw Points', true); settings.addBoolean('Draw Polygons', true); @@ -342,21 +350,20 @@ function setupUI() { settings.addRange('FPS', 0, 100, 0, 1); } -async function setupCanvas(input) { - // setup canvas object to same size as input as camera resolution may change - const canvas = document.getElementById('canvas'); - canvas.width = input.width; - canvas.height = input.height; - return canvas; -} - // eslint-disable-next-line no-unused-vars async function setupCamera() { - log('Setting up camera'); - // setup webcam. note that navigator.mediaDevices requires that page is accessed via https + if (ui.busy) return null; + ui.busy = true; const video = document.getElementById('video'); + const canvas = document.getElementById('canvas'); + const output = document.getElementById('log'); + const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false; + log('Setting up camera: live:', live); + // setup webcam. note that navigator.mediaDevices requires that page is accessed via https if (!navigator.mediaDevices) { - document.getElementById('log').innerText = 'Video not supported'; + const msg = 'Camera access not supported'; + output.innerText = msg; + log(msg); return null; } const stream = await navigator.mediaDevices.getUserMedia({ @@ -365,11 +372,15 @@ async function setupCamera() { }); video.srcObject = stream; return new Promise((resolve) => { - video.onloadedmetadata = () => { + video.onloadeddata = async () => { video.width = video.videoWidth; video.height = video.videoHeight; - video.play(); - video.pause(); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + if (live) video.play(); + ui.busy = false; + // do once more because onresize events can be delayed or skipped + if (video.width !== window.innerWidth) await setupCamera(); resolve(video); }; }); @@ -387,16 +398,15 @@ async function setupImage() { } async function main() { - log('Human starting ...'); + log('Human demo starting ...'); // setup ui control panel await setupUI(); // setup webcam - const input = await setupCamera(); + await setupCamera(); // or setup image // const input = await setupImage(); // setup output canvas from input object - await setupCanvas(input); const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`; document.getElementById('log').innerText = msg; @@ -407,4 +417,4 @@ async function main() { } window.onload = main; -window.onresize = main; +window.onresize = setupCamera; diff --git a/demo/demo-iife.html b/demo/demo-iife.html index 1d1a8480..0296c31c 100644 --- a/demo/demo-iife.html +++ b/demo/demo-iife.html @@ -1,7 +1,6 @@ -
@@ -10,4 +9,414 @@
Starting Human library
+ diff --git a/demo/demo-node.js b/demo/demo-node.js index 7862fc68..5771a989 100644 --- a/demo/demo-node.js +++ b/demo/demo-node.js @@ -2,7 +2,7 @@ const tf = require('@tensorflow/tfjs-node'); const fs = require('fs'); const process = require('process'); const console = require('console'); -const human = require('..'); // this would be '@vladmandic/human' +const human = require('..'); // this resolves to project root which is '@vladmandic/human' const logger = new console.Console({ stdout: process.stdout, @@ -24,6 +24,8 @@ const logger = new console.Console({ }); const config = { + backend: 'tensorflow', + console: true, face: { enabled: false, detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 }, diff --git a/package.json b/package.json index 887c7227..ffd05191 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.3.3", "description": "human: 3D Face Detection, Iris Tracking and Age & Gender Prediction", "sideEffects": false, - "main": "dist/human.cjs", + "main": "dist/human-nobundle.cjs", "module": "dist/human.esm.js", "browser": "dist/human.esm.js", "author": "Vladimir Mandic ", diff --git a/src/config.js b/src/config.js index 95c58a43..90865bb1 100644 --- a/src/config.js +++ b/src/config.js @@ -1,4 +1,6 @@ export default { + backend: 'webgl', + console: true, face: { enabled: true, // refers to detector, but since all other face modules rely on detector, it should be a global detector: { diff --git a/src/index.js b/src/index.js index bdb3819e..1ea656b2 100644 --- a/src/index.js +++ b/src/index.js @@ -19,6 +19,10 @@ const models = { gender: null, emotion: null, }; +const now = () => { + if (typeof performance !== 'undefined') return performance.now(); + return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000); +}; const log = (...msg) => { // eslint-disable-next-line no-console @@ -44,11 +48,31 @@ function mergeDeep(...objects) { }, {}); } +function sanity(input) { + if (!input) return 'input is not defined'; + const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); + if (!width || (width === 0)) return 'input is empty'; + if (input.readyState && (input.readyState <= 2)) return 'input is not ready'; + try { + tf.getBackend(); + } catch { + return 'backend not loaded'; + } + return null; +} + async function detect(input, userConfig) { + config = mergeDeep(defaults, userConfig); + + // sanity checks + const error = sanity(input); + if (error) { + log(error, input); + return { error }; + } + // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve) => { - config = mergeDeep(defaults, userConfig); - // check number of loaded models const loadedModels = Object.values(models).filter((a) => a).length; if (loadedModels === 0) log('Human library starting'); @@ -78,35 +102,40 @@ async function detect(input, userConfig) { let timeStamp; // run posenet - timeStamp = performance.now(); + timeStamp = now(); tf.engine().startScope(); const poseRes = config.body.enabled ? await models.posenet.estimatePoses(input, config.body) : []; tf.engine().endScope(); - perf.body = Math.trunc(performance.now() - timeStamp); + perf.body = Math.trunc(now() - timeStamp); // run handpose - timeStamp = performance.now(); + timeStamp = now(); tf.engine().startScope(); const handRes = config.hand.enabled ? await models.handpose.estimateHands(input, config.hand) : []; tf.engine().endScope(); - perf.hand = Math.trunc(performance.now() - timeStamp); + perf.hand = Math.trunc(now() - timeStamp); // run facemesh, includes blazeface and iris const faceRes = []; if (config.face.enabled) { - timeStamp = performance.now(); + timeStamp = now(); tf.engine().startScope(); const faces = await models.facemesh.estimateFaces(input, config.face); - perf.face = Math.trunc(performance.now() - timeStamp); + perf.face = Math.trunc(now() - timeStamp); for (const face of faces) { + // is something went wrong, skip the face + if (!face.image || face.image.isDisposedInternal) { + log('face object is disposed:', face.image); + continue; + } // run ssr-net age & gender, inherits face from blazeface - timeStamp = performance.now(); + timeStamp = now(); const ssrData = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {}; - perf.agegender = Math.trunc(performance.now() - timeStamp); + perf.agegender = Math.trunc(now() - timeStamp); // run emotion, inherits face from blazeface - timeStamp = performance.now(); + timeStamp = now(); const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {}; - perf.emotion = Math.trunc(performance.now() - timeStamp); + perf.emotion = Math.trunc(now() - timeStamp); face.image.dispose(); // calculate iris distance // iris: array[ bottom, left, top, right, center ]