diff --git a/README.md b/README.md
index 4055c666..ad254f8f 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
Compatible with Browser, WebWorker and NodeJS execution!
(and maybe with React-Native as it doesn't use any DOM objects)
-*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations*
+*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations and planned enhancements*
*Suggestions are welcome!*
@@ -124,8 +124,8 @@ And then use with:
const human = require('@vladmandic/human'); // points to @vladmandic/human/dist/human.cjs
```
-
Since NodeJS projects load `weights` from local filesystem instead of using `http` calls, you must modify default configuration to include correct paths with `file://` prefix
+
For example:
```js
const config = {
@@ -213,7 +213,6 @@ Note that user object and default configuration are merged using deep-merge, so
Configurtion object is large, but typically you only need to modify few values:
- `enabled`: Choose which models to use
-- `skipFrames`: Must be set to 0 for static images
- `modelPath`: Update as needed to reflect your application's relative path
@@ -234,8 +233,9 @@ config = {
inputSize: 256, // fixed value: 128 for front and 256 for 'back'
maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance
skipFrames: 10, // how many frames to go without re-running the face bounding box detector
+ // only used for video inputs, ignored for static inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis
- // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
+ // as the face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
@@ -256,7 +256,7 @@ config = {
modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki'
// which determines training set for model
inputSize: 64, // fixed value
- skipFrames: 10, // how many frames to go without re-running the detector
+ skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
},
gender: {
enabled: true,
@@ -267,7 +267,7 @@ config = {
enabled: true,
inputSize: 64, // fixed value
minConfidence: 0.5, // threshold for discarding a prediction
- skipFrames: 10, // how many frames to go without re-running the detector
+ skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
useGrayscale: true, // convert image to grayscale before prediction or use highest channel
modelPath: '../models/emotion/model.json',
},
@@ -285,8 +285,9 @@ config = {
enabled: true,
inputSize: 256, // fixed value
skipFrames: 10, // how many frames to go without re-running the hand bounding box detector
+ // only used for video inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
- // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
+ // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
diff --git a/config.js b/config.js
index eb49ce04..505d942a 100644
--- a/config.js
+++ b/config.js
@@ -16,7 +16,7 @@ export default {
// 'front' is optimized for large faces such as front-facing camera and 'back' is optimized for distanct faces.
inputSize: 256, // fixed value: 128 for front and 256 for 'back'
maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance
- skipFrames: 10, // how many frames to go without re-running the face bounding box detector
+ skipFrames: 10, // how many frames to go without re-running the face bounding box detector, only used for video inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction
@@ -39,7 +39,7 @@ export default {
modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki'
// which determines training set for model
inputSize: 64, // fixed value
- skipFrames: 10, // how many frames to go without re-running the detector
+ skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
},
gender: {
enabled: true,
@@ -67,9 +67,9 @@ export default {
hand: {
enabled: true,
inputSize: 256, // fixed value
- skipFrames: 10, // how many frames to go without re-running the hand bounding box detector
+ skipFrames: 10, // how many frames to go without re-running the hand bounding box detector, only used for video inputs
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
- // as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
+ // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
diff --git a/demo/browser.js b/demo/browser.js
index 82c342bd..4e2a0d39 100644
--- a/demo/browser.js
+++ b/demo/browser.js
@@ -180,12 +180,6 @@ function runHumanDetect(input, canvas) {
// main processing function when input is image, can use direct invocation or web worker
async function processImage(input) {
- // must be zero for images
- config.face.detector.skipFrames = 0;
- config.face.emotion.skipFrames = 0;
- config.face.age.skipFrames = 0;
- config.hand.skipFrames = 0;
-
timeStamp = performance.now();
return new Promise((resolve) => {
const image = document.getElementById('image');
@@ -234,7 +228,7 @@ async function detectVideo() {
// just initialize everything and call main function
async function detectSampleImages() {
- ui.baseFont = ui.baseFontProto.replace(/{size}/, `${ui.columns}rem`);
+ ui.baseFont = ui.baseFontProto.replace(/{size}/, `${1.2 * ui.columns}rem`);
ui.baseLineHeight = ui.baseLineHeightProto * ui.columns;
document.getElementById('canvas').style.display = 'none';
document.getElementById('samples').style.display = 'block';
@@ -244,6 +238,7 @@ async function detectSampleImages() {
function setupMenu() {
menu = new Menu(document.body);
+ menu.addTitle('...');
menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt));
menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
@@ -297,7 +292,6 @@ function setupMenu() {
menu.addBool('Fill Polygons', ui, 'fillPolygons');
menu.addHTML('
');
- menu.addValue('State', '');
menu.addChart('FPS', 'FPS');
}
diff --git a/demo/menu.js b/demo/menu.js
index c6f47c76..f6274a45 100644
--- a/demo/menu.js
+++ b/demo/menu.js
@@ -1,19 +1,22 @@
const css = `
- .menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); }
+ .menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); box-shadow: 0 0 8px dimgrey; }
+ .menu-container:hover { box-shadow: 0 0 8px lightgrey; }
.menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; }
- .menu-title { padding: 0; }
+ .menu-title { text-align: right; cursor: pointer; }
.menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) }
- .menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
+ .menu-label { padding: 0; }
.menu-chart-title { align-items: center; }
.menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; }
.menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; }
- .menu-button:hover { background: lightgreen; }
+ .menu-button:hover { background: lightgreen; box-shadow: 4px 4px 4px 0 black; }
+ .menu-button:focus { outline: none; }
.menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; }
.menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
.menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
+ .menu-checkbox-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
input[type=checkbox] { visibility: hidden; }
input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; }
@@ -45,6 +48,7 @@ class Menu {
this.menu = createElem(parent);
this._id = 0;
this._maxFPS = 0;
+ this.hidden = 0;
}
get newID() {
@@ -64,9 +68,22 @@ class Menu {
return this.menu.offsetHeight;
}
+ async addTitle(title) {
+ const el = document.createElement('div');
+ el.className = 'menu-title';
+ el.id = this.newID;
+ el.innerHTML = title;
+ this.menu.appendChild(el);
+ el.addEventListener('click', () => {
+ this.hidden = !this.hidden;
+ const all = document.getElementsByClassName('menu');
+ for (const item of all) item.style.display = this.hidden ? 'none' : 'flex';
+ });
+ }
+
async addLabel(title) {
const el = document.createElement('div');
- el.className = 'menu menu-title';
+ el.className = 'menu menu-label';
el.id = this.newID;
el.innerHTML = title;
this.menu.appendChild(el);
@@ -75,9 +92,9 @@ class Menu {
async addBool(title, object, variable, callback) {
const el = document.createElement('div');
el.className = 'menu';
- el.innerHTML = `${title}`;
+ el.innerHTML = `${title}`;
this.menu.appendChild(el);
- document.getElementById(this.ID).addEventListener('change', (evt) => {
+ el.addEventListener('change', (evt) => {
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
});
@@ -88,7 +105,7 @@ class Menu {
el.className = 'menu';
el.innerHTML = `${title}`;
this.menu.appendChild(el);
- document.getElementById(this.ID).addEventListener('change', (evt) => {
+ el.addEventListener('change', (evt) => {
object[variable] = evt.target.value;
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target.value);
@@ -106,11 +123,14 @@ class Menu {
async addButton(titleOn, titleOff, callback) {
const el = document.createElement('button');
el.className = 'menu menu-button';
+ el.style.fontFamily = document.body.style.fontFamily;
+ el.style.fontSize = document.body.style.fontSize;
+ el.style.fontVariant = document.body.style.fontVariant;
el.type = 'button';
el.id = this.newID;
el.innerText = titleOn;
this.menu.appendChild(el);
- document.getElementById(this.ID).addEventListener('click', () => {
+ el.addEventListener('click', () => {
if (el.innerText === titleOn) el.innerText = titleOff;
else el.innerText = titleOn;
if (callback) callback(el.innerText !== titleOn);
diff --git a/demo/node.js b/demo/node.js
index a1232ea1..a06e5fd6 100644
--- a/demo/node.js
+++ b/demo/node.js
@@ -27,21 +27,15 @@ const config = {
backend: 'tensorflow',
console: true,
face: {
- enabled: false,
- detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
- mesh: { enabled: true, modelPath: 'file://models/facemesh/model.json', inputSize: 192 },
- iris: { enabled: true, modelPath: 'file://models/iris/model.json', inputSize: 192 },
- age: { enabled: true, modelPath: 'file://models/ssrnet-age/imdb/model.json', inputSize: 64, skipFrames: 5 },
- gender: { enabled: true, modelPath: 'file://models/ssrnet-gender/imdb/model.json' },
+ detector: { modelPath: 'file://models/blazeface/back/model.json' },
+ mesh: { modelPath: 'file://models/facemesh/model.json' },
+ iris: { modelPath: 'file://models/iris/model.json' },
+ age: { modelPath: 'file://models/ssrnet-age/imdb/model.json' },
+ gender: { modelPath: 'file://models/ssrnet-gender/imdb/model.json' },
+ emotion: { modelPath: 'file://models/emotion/model.json' },
},
- body: { enabled: true, modelPath: 'file://models/posenet/model.json', inputResolution: 257, outputStride: 16, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
+ body: { modelPath: 'file://models/posenet/model.json' },
hand: {
- enabled: false,
- inputSize: 256,
- skipFrames: 10,
- minConfidence: 0.8,
- iouThreshold: 0.3,
- scoreThreshold: 0.75,
detector: { anchors: 'file://models/handdetect/anchors.json', modelPath: 'file://models/handdetect/model.json' },
skeleton: { modelPath: 'file://models/handskeleton/model.json' },
},
diff --git a/package.json b/package.json
index 62a73714..a0ef0955 100644
--- a/package.json
+++ b/package.json
@@ -37,12 +37,12 @@
"rimraf": "^3.0.2"
},
"scripts": {
- "start": "node --trace-warnings --trace-uncaught --no-deprecation demo/node.js",
+ "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js",
"lint": "eslint src/*.js demo/*.js",
- "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/index.js",
- "build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/index.js",
- "build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/index.js",
- "build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/index.js",
+ "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/human.js",
+ "build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js",
+ "build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js",
+ "build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/human.js",
"build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && ls -l dist/",
"update": "npm update --depth 20 && npm dedupe && npm prune && npm audit",
"changelog": "node changelog.js"
diff --git a/src/emotion/emotion.js b/src/emotion/emotion.js
index bb5ec2a7..89d36ca9 100644
--- a/src/emotion/emotion.js
+++ b/src/emotion/emotion.js
@@ -22,11 +22,11 @@ async function load(config) {
}
async function predict(image, config) {
- frame += 1;
- if (frame >= config.face.emotion.skipFrames) {
- frame = 0;
+ if (frame < config.face.emotion.skipFrames) {
+ frame += 1;
return last;
}
+ frame = 0;
const enhance = tf.tidy(() => {
if (image instanceof tf.Tensor) {
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
diff --git a/src/handpose/handpose.js b/src/handpose/handpose.js
index 58136fac..1326cde6 100644
--- a/src/handpose/handpose.js
+++ b/src/handpose/handpose.js
@@ -9,7 +9,7 @@ class HandPose {
}
async estimateHands(input, config) {
- this.maxContinuousChecks = config.skipFrames;
+ this.skipFrames = config.skipFrames;
this.detectionConfidence = config.minConfidence;
this.maxHands = config.maxHands;
const image = tf.tidy(() => {
diff --git a/src/index.js b/src/human.js
similarity index 86%
rename from src/index.js
rename to src/human.js
index 2755074c..4b1b9d31 100644
--- a/src/index.js
+++ b/src/human.js
@@ -21,6 +21,11 @@ const models = {
emotion: null,
};
+const override = {
+ face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } },
+ hand: { skipFrames: 0 },
+};
+
// helper function: gets elapsed time on both browser and nodejs
const now = () => {
if (typeof performance !== 'undefined') return performance.now();
@@ -66,9 +71,16 @@ function mergeDeep(...objects) {
function sanity(input) {
if (!input) return 'input is not defined';
- const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
- if (!width || (width === 0)) return 'input is empty';
- if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
+ if (tf.ENV.flags.IS_BROWSER && (input instanceof ImageData || input instanceof HTMLImageElement || input instanceof HTMLCanvasElement || input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) {
+ const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
+ if (!width || (width === 0)) return 'input is empty';
+ }
+ if (tf.ENV.flags.IS_BROWSER && (input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) {
+ if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
+ }
+ if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
+ return 'input must be a tensor';
+ }
try {
tf.getBackend();
} catch {
@@ -93,7 +105,8 @@ async function detect(input, userConfig = {}) {
let timeStamp;
timeStamp = now();
- config = mergeDeep(defaults, userConfig);
+ const shouldOverride = tf.ENV.flags.IS_NODE || (tf.ENV.flags.IS_BROWSER && !((input instanceof HTMLVideoElement) || (input instanceof HTMLMediaElement)));
+ config = mergeDeep(defaults, userConfig, shouldOverride ? override : {});
perf.config = Math.trunc(now() - timeStamp);
// sanity checks
@@ -222,3 +235,5 @@ exports.handpose = handpose;
exports.tf = tf;
exports.version = app.version;
exports.state = state;
+
+// Error: Failed to compile fragment shader