From 7a6741e3b45580502fc223e8e2b9e358f9eeea7e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 18 Oct 2020 08:07:45 -0400 Subject: [PATCH] autodetect skipFrames --- README.md | 15 ++++++++------- config.js | 8 ++++---- demo/browser.js | 10 ++-------- demo/menu.js | 38 +++++++++++++++++++++++++++++--------- demo/node.js | 20 +++++++------------- package.json | 10 +++++----- src/emotion/emotion.js | 6 +++--- src/handpose/handpose.js | 2 +- src/{index.js => human.js} | 23 +++++++++++++++++++---- 9 files changed, 78 insertions(+), 54 deletions(-) rename src/{index.js => human.js} (86%) diff --git a/README.md b/README.md index 4055c666..ad254f8f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Compatible with Browser, WebWorker and NodeJS execution! (and maybe with React-Native as it doesn't use any DOM objects) -*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations* +*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations and planned enhancements* *Suggestions are welcome!* @@ -124,8 +124,8 @@ And then use with: const human = require('@vladmandic/human'); // points to @vladmandic/human/dist/human.cjs ``` - Since NodeJS projects load `weights` from local filesystem instead of using `http` calls, you must modify default configuration to include correct paths with `file://` prefix + For example: ```js const config = { @@ -213,7 +213,6 @@ Note that user object and default configuration are merged using deep-merge, so Configurtion object is large, but typically you only need to modify few values: - `enabled`: Choose which models to use -- `skipFrames`: Must be set to 0 for static images - `modelPath`: Update as needed to reflect your application's relative path @@ -234,8 +233,9 @@ config = { inputSize: 256, // fixed value: 128 for front and 256 for 'back' maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance skipFrames: 10, // how many frames to go without re-running the face bounding box detector + // only used for video inputs, ignored for static inputs // if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis - // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) + // as the face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.5, // threshold for discarding a prediction iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression @@ -256,7 +256,7 @@ config = { modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki' // which determines training set for model inputSize: 64, // fixed value - skipFrames: 10, // how many frames to go without re-running the detector + skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs }, gender: { enabled: true, @@ -267,7 +267,7 @@ config = { enabled: true, inputSize: 64, // fixed value minConfidence: 0.5, // threshold for discarding a prediction - skipFrames: 10, // how many frames to go without re-running the detector + skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs useGrayscale: true, // convert image to grayscale before prediction or use highest channel modelPath: '../models/emotion/model.json', }, @@ -285,8 +285,9 @@ config = { enabled: true, inputSize: 256, // fixed value skipFrames: 10, // how many frames to go without re-running the hand bounding box detector + // only used for video inputs // if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis - // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) + // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.5, // threshold for discarding a prediction iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression diff --git a/config.js b/config.js index eb49ce04..505d942a 100644 --- a/config.js +++ b/config.js @@ -16,7 +16,7 @@ export default { // 'front' is optimized for large faces such as front-facing camera and 'back' is optimized for distanct faces. inputSize: 256, // fixed value: 128 for front and 256 for 'back' maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance - skipFrames: 10, // how many frames to go without re-running the face bounding box detector + skipFrames: 10, // how many frames to go without re-running the face bounding box detector, only used for video inputs // if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.5, // threshold for discarding a prediction @@ -39,7 +39,7 @@ export default { modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki' // which determines training set for model inputSize: 64, // fixed value - skipFrames: 10, // how many frames to go without re-running the detector + skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs }, gender: { enabled: true, @@ -67,9 +67,9 @@ export default { hand: { enabled: true, inputSize: 256, // fixed value - skipFrames: 10, // how many frames to go without re-running the hand bounding box detector + skipFrames: 10, // how many frames to go without re-running the hand bounding box detector, only used for video inputs // if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis - // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) + // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.5, // threshold for discarding a prediction iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression diff --git a/demo/browser.js b/demo/browser.js index 82c342bd..4e2a0d39 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -180,12 +180,6 @@ function runHumanDetect(input, canvas) { // main processing function when input is image, can use direct invocation or web worker async function processImage(input) { - // must be zero for images - config.face.detector.skipFrames = 0; - config.face.emotion.skipFrames = 0; - config.face.age.skipFrames = 0; - config.hand.skipFrames = 0; - timeStamp = performance.now(); return new Promise((resolve) => { const image = document.getElementById('image'); @@ -234,7 +228,7 @@ async function detectVideo() { // just initialize everything and call main function async function detectSampleImages() { - ui.baseFont = ui.baseFontProto.replace(/{size}/, `${ui.columns}rem`); + ui.baseFont = ui.baseFontProto.replace(/{size}/, `${1.2 * ui.columns}rem`); ui.baseLineHeight = ui.baseLineHeightProto * ui.columns; document.getElementById('canvas').style.display = 'none'; document.getElementById('samples').style.display = 'block'; @@ -244,6 +238,7 @@ async function detectSampleImages() { function setupMenu() { menu = new Menu(document.body); + menu.addTitle('...'); menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt)); menu.addButton('Process Images', 'Process Images', () => detectSampleImages()); @@ -297,7 +292,6 @@ function setupMenu() { menu.addBool('Fill Polygons', ui, 'fillPolygons'); menu.addHTML('
'); - menu.addValue('State', ''); menu.addChart('FPS', 'FPS'); } diff --git a/demo/menu.js b/demo/menu.js index c6f47c76..f6274a45 100644 --- a/demo/menu.js +++ b/demo/menu.js @@ -1,19 +1,22 @@ const css = ` - .menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); } + .menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); box-shadow: 0 0 8px dimgrey; } + .menu-container:hover { box-shadow: 0 0 8px lightgrey; } .menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; } - .menu-title { padding: 0; } + .menu-title { text-align: right; cursor: pointer; } .menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) } - .menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; } + .menu-label { padding: 0; } .menu-chart-title { align-items: center; } .menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; } .menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; } - .menu-button:hover { background: lightgreen; } + .menu-button:hover { background: lightgreen; box-shadow: 4px 4px 4px 0 black; } + .menu-button:focus { outline: none; } .menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; } .menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } .menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } + .menu-checkbox-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; } input[type=checkbox] { visibility: hidden; } input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; } @@ -45,6 +48,7 @@ class Menu { this.menu = createElem(parent); this._id = 0; this._maxFPS = 0; + this.hidden = 0; } get newID() { @@ -64,9 +68,22 @@ class Menu { return this.menu.offsetHeight; } + async addTitle(title) { + const el = document.createElement('div'); + el.className = 'menu-title'; + el.id = this.newID; + el.innerHTML = title; + this.menu.appendChild(el); + el.addEventListener('click', () => { + this.hidden = !this.hidden; + const all = document.getElementsByClassName('menu'); + for (const item of all) item.style.display = this.hidden ? 'none' : 'flex'; + }); + } + async addLabel(title) { const el = document.createElement('div'); - el.className = 'menu menu-title'; + el.className = 'menu menu-label'; el.id = this.newID; el.innerHTML = title; this.menu.appendChild(el); @@ -75,9 +92,9 @@ class Menu { async addBool(title, object, variable, callback) { const el = document.createElement('div'); el.className = 'menu'; - el.innerHTML = `
${title}`; + el.innerHTML = `
${title}`; this.menu.appendChild(el); - document.getElementById(this.ID).addEventListener('change', (evt) => { + el.addEventListener('change', (evt) => { object[variable] = evt.target.checked; if (callback) callback(evt.target.checked); }); @@ -88,7 +105,7 @@ class Menu { el.className = 'menu'; el.innerHTML = `${title}`; this.menu.appendChild(el); - document.getElementById(this.ID).addEventListener('change', (evt) => { + el.addEventListener('change', (evt) => { object[variable] = evt.target.value; evt.target.setAttribute('value', evt.target.value); if (callback) callback(evt.target.value); @@ -106,11 +123,14 @@ class Menu { async addButton(titleOn, titleOff, callback) { const el = document.createElement('button'); el.className = 'menu menu-button'; + el.style.fontFamily = document.body.style.fontFamily; + el.style.fontSize = document.body.style.fontSize; + el.style.fontVariant = document.body.style.fontVariant; el.type = 'button'; el.id = this.newID; el.innerText = titleOn; this.menu.appendChild(el); - document.getElementById(this.ID).addEventListener('click', () => { + el.addEventListener('click', () => { if (el.innerText === titleOn) el.innerText = titleOff; else el.innerText = titleOn; if (callback) callback(el.innerText !== titleOn); diff --git a/demo/node.js b/demo/node.js index a1232ea1..a06e5fd6 100644 --- a/demo/node.js +++ b/demo/node.js @@ -27,21 +27,15 @@ const config = { backend: 'tensorflow', console: true, face: { - enabled: false, - detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 }, - mesh: { enabled: true, modelPath: 'file://models/facemesh/model.json', inputSize: 192 }, - iris: { enabled: true, modelPath: 'file://models/iris/model.json', inputSize: 192 }, - age: { enabled: true, modelPath: 'file://models/ssrnet-age/imdb/model.json', inputSize: 64, skipFrames: 5 }, - gender: { enabled: true, modelPath: 'file://models/ssrnet-gender/imdb/model.json' }, + detector: { modelPath: 'file://models/blazeface/back/model.json' }, + mesh: { modelPath: 'file://models/facemesh/model.json' }, + iris: { modelPath: 'file://models/iris/model.json' }, + age: { modelPath: 'file://models/ssrnet-age/imdb/model.json' }, + gender: { modelPath: 'file://models/ssrnet-gender/imdb/model.json' }, + emotion: { modelPath: 'file://models/emotion/model.json' }, }, - body: { enabled: true, modelPath: 'file://models/posenet/model.json', inputResolution: 257, outputStride: 16, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 }, + body: { modelPath: 'file://models/posenet/model.json' }, hand: { - enabled: false, - inputSize: 256, - skipFrames: 10, - minConfidence: 0.8, - iouThreshold: 0.3, - scoreThreshold: 0.75, detector: { anchors: 'file://models/handdetect/anchors.json', modelPath: 'file://models/handdetect/model.json' }, skeleton: { modelPath: 'file://models/handskeleton/model.json' }, }, diff --git a/package.json b/package.json index 62a73714..a0ef0955 100644 --- a/package.json +++ b/package.json @@ -37,12 +37,12 @@ "rimraf": "^3.0.2" }, "scripts": { - "start": "node --trace-warnings --trace-uncaught --no-deprecation demo/node.js", + "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js", "lint": "eslint src/*.js demo/*.js", - "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/index.js", - "build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/index.js", - "build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/index.js", - "build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/index.js", + "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/human.js", + "build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js", + "build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js", + "build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/human.js", "build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && ls -l dist/", "update": "npm update --depth 20 && npm dedupe && npm prune && npm audit", "changelog": "node changelog.js" diff --git a/src/emotion/emotion.js b/src/emotion/emotion.js index bb5ec2a7..89d36ca9 100644 --- a/src/emotion/emotion.js +++ b/src/emotion/emotion.js @@ -22,11 +22,11 @@ async function load(config) { } async function predict(image, config) { - frame += 1; - if (frame >= config.face.emotion.skipFrames) { - frame = 0; + if (frame < config.face.emotion.skipFrames) { + frame += 1; return last; } + frame = 0; const enhance = tf.tidy(() => { if (image instanceof tf.Tensor) { const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false); diff --git a/src/handpose/handpose.js b/src/handpose/handpose.js index 58136fac..1326cde6 100644 --- a/src/handpose/handpose.js +++ b/src/handpose/handpose.js @@ -9,7 +9,7 @@ class HandPose { } async estimateHands(input, config) { - this.maxContinuousChecks = config.skipFrames; + this.skipFrames = config.skipFrames; this.detectionConfidence = config.minConfidence; this.maxHands = config.maxHands; const image = tf.tidy(() => { diff --git a/src/index.js b/src/human.js similarity index 86% rename from src/index.js rename to src/human.js index 2755074c..4b1b9d31 100644 --- a/src/index.js +++ b/src/human.js @@ -21,6 +21,11 @@ const models = { emotion: null, }; +const override = { + face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, + hand: { skipFrames: 0 }, +}; + // helper function: gets elapsed time on both browser and nodejs const now = () => { if (typeof performance !== 'undefined') return performance.now(); @@ -66,9 +71,16 @@ function mergeDeep(...objects) { function sanity(input) { if (!input) return 'input is not defined'; - const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); - if (!width || (width === 0)) return 'input is empty'; - if (input.readyState && (input.readyState <= 2)) return 'input is not ready'; + if (tf.ENV.flags.IS_BROWSER && (input instanceof ImageData || input instanceof HTMLImageElement || input instanceof HTMLCanvasElement || input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) { + const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); + if (!width || (width === 0)) return 'input is empty'; + } + if (tf.ENV.flags.IS_BROWSER && (input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) { + if (input.readyState && (input.readyState <= 2)) return 'input is not ready'; + } + if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) { + return 'input must be a tensor'; + } try { tf.getBackend(); } catch { @@ -93,7 +105,8 @@ async function detect(input, userConfig = {}) { let timeStamp; timeStamp = now(); - config = mergeDeep(defaults, userConfig); + const shouldOverride = tf.ENV.flags.IS_NODE || (tf.ENV.flags.IS_BROWSER && !((input instanceof HTMLVideoElement) || (input instanceof HTMLMediaElement))); + config = mergeDeep(defaults, userConfig, shouldOverride ? override : {}); perf.config = Math.trunc(now() - timeStamp); // sanity checks @@ -222,3 +235,5 @@ exports.handpose = handpose; exports.tf = tf; exports.version = app.version; exports.state = state; + +// Error: Failed to compile fragment shader