mirror of https://github.com/vladmandic/human
autodetect skipFrames
Vladimir Mandic
parent
08c55327bb
commit
d44ff5dbb2
15
README.md
15
README.md
|
|
@ -12,7 +12,7 @@
|
|||
Compatible with Browser, WebWorker and NodeJS execution!
|
||||
(and maybe with React-Native as it doesn't use any DOM objects)
|
||||
|
||||
*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations*
|
||||
*This is a pre-release project, see [issues](https://github.com/vladmandic/human/issues) for list of known limitations and planned enhancements*
|
||||
|
||||
*Suggestions are welcome!*
|
||||
|
||||
|
|
@ -124,8 +124,8 @@ And then use with:
|
|||
const human = require('@vladmandic/human'); // points to @vladmandic/human/dist/human.cjs
|
||||
```
|
||||
|
||||
|
||||
Since NodeJS projects load `weights` from local filesystem instead of using `http` calls, you must modify default configuration to include correct paths with `file://` prefix
|
||||
|
||||
For example:
|
||||
```js
|
||||
const config = {
|
||||
|
|
@ -213,7 +213,6 @@ Note that user object and default configuration are merged using deep-merge, so
|
|||
Configurtion object is large, but typically you only need to modify few values:
|
||||
|
||||
- `enabled`: Choose which models to use
|
||||
- `skipFrames`: Must be set to 0 for static images
|
||||
- `modelPath`: Update as needed to reflect your application's relative path
|
||||
|
||||
|
||||
|
|
@ -234,8 +233,9 @@ config = {
|
|||
inputSize: 256, // fixed value: 128 for front and 256 for 'back'
|
||||
maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
skipFrames: 10, // how many frames to go without re-running the face bounding box detector
|
||||
// only used for video inputs, ignored for static inputs
|
||||
// if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis
|
||||
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
// as the face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
|
|
@ -256,7 +256,7 @@ config = {
|
|||
modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki'
|
||||
// which determines training set for model
|
||||
inputSize: 64, // fixed value
|
||||
skipFrames: 10, // how many frames to go without re-running the detector
|
||||
skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
|
||||
},
|
||||
gender: {
|
||||
enabled: true,
|
||||
|
|
@ -267,7 +267,7 @@ config = {
|
|||
enabled: true,
|
||||
inputSize: 64, // fixed value
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
skipFrames: 10, // how many frames to go without re-running the detector
|
||||
skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
|
||||
useGrayscale: true, // convert image to grayscale before prediction or use highest channel
|
||||
modelPath: '../models/emotion/model.json',
|
||||
},
|
||||
|
|
@ -285,8 +285,9 @@ config = {
|
|||
enabled: true,
|
||||
inputSize: 256, // fixed value
|
||||
skipFrames: 10, // how many frames to go without re-running the hand bounding box detector
|
||||
// only used for video inputs
|
||||
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
|
||||
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
// as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ export default {
|
|||
// 'front' is optimized for large faces such as front-facing camera and 'back' is optimized for distanct faces.
|
||||
inputSize: 256, // fixed value: 128 for front and 256 for 'back'
|
||||
maxFaces: 10, // maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
skipFrames: 10, // how many frames to go without re-running the face bounding box detector
|
||||
skipFrames: 10, // how many frames to go without re-running the face bounding box detector, only used for video inputs
|
||||
// if model is running st 25 FPS, we can re-use existing bounding box for updated face mesh analysis
|
||||
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
|
|
@ -39,7 +39,7 @@ export default {
|
|||
modelPath: '../models/ssrnet-age/imdb/model.json', // can be 'imdb' or 'wiki'
|
||||
// which determines training set for model
|
||||
inputSize: 64, // fixed value
|
||||
skipFrames: 10, // how many frames to go without re-running the detector
|
||||
skipFrames: 10, // how many frames to go without re-running the detector, only used for video inputs
|
||||
},
|
||||
gender: {
|
||||
enabled: true,
|
||||
|
|
@ -67,9 +67,9 @@ export default {
|
|||
hand: {
|
||||
enabled: true,
|
||||
inputSize: 256, // fixed value
|
||||
skipFrames: 10, // how many frames to go without re-running the hand bounding box detector
|
||||
skipFrames: 10, // how many frames to go without re-running the hand bounding box detector, only used for video inputs
|
||||
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
|
||||
// as face probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
// as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.5, // threshold for discarding a prediction
|
||||
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
scoreThreshold: 0.7, // threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
|
|
|
|||
|
|
@ -180,12 +180,6 @@ function runHumanDetect(input, canvas) {
|
|||
|
||||
// main processing function when input is image, can use direct invocation or web worker
|
||||
async function processImage(input) {
|
||||
// must be zero for images
|
||||
config.face.detector.skipFrames = 0;
|
||||
config.face.emotion.skipFrames = 0;
|
||||
config.face.age.skipFrames = 0;
|
||||
config.hand.skipFrames = 0;
|
||||
|
||||
timeStamp = performance.now();
|
||||
return new Promise((resolve) => {
|
||||
const image = document.getElementById('image');
|
||||
|
|
@ -234,7 +228,7 @@ async function detectVideo() {
|
|||
|
||||
// just initialize everything and call main function
|
||||
async function detectSampleImages() {
|
||||
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${ui.columns}rem`);
|
||||
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${1.2 * ui.columns}rem`);
|
||||
ui.baseLineHeight = ui.baseLineHeightProto * ui.columns;
|
||||
document.getElementById('canvas').style.display = 'none';
|
||||
document.getElementById('samples').style.display = 'block';
|
||||
|
|
@ -244,6 +238,7 @@ async function detectSampleImages() {
|
|||
|
||||
function setupMenu() {
|
||||
menu = new Menu(document.body);
|
||||
menu.addTitle('...');
|
||||
menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt));
|
||||
menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
|
||||
|
||||
|
|
@ -297,7 +292,6 @@ function setupMenu() {
|
|||
menu.addBool('Fill Polygons', ui, 'fillPolygons');
|
||||
|
||||
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
|
||||
menu.addValue('State', '');
|
||||
menu.addChart('FPS', 'FPS');
|
||||
}
|
||||
|
||||
|
|
|
|||
38
demo/menu.js
38
demo/menu.js
|
|
@ -1,19 +1,22 @@
|
|||
const css = `
|
||||
.menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); }
|
||||
.menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); box-shadow: 0 0 8px dimgrey; }
|
||||
.menu-container:hover { box-shadow: 0 0 8px lightgrey; }
|
||||
.menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; }
|
||||
.menu-title { padding: 0; }
|
||||
.menu-title { text-align: right; cursor: pointer; }
|
||||
.menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) }
|
||||
.menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
|
||||
.menu-label { padding: 0; }
|
||||
|
||||
.menu-chart-title { align-items: center; }
|
||||
.menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; }
|
||||
|
||||
.menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; }
|
||||
.menu-button:hover { background: lightgreen; }
|
||||
.menu-button:hover { background: lightgreen; box-shadow: 4px 4px 4px 0 black; }
|
||||
.menu-button:focus { outline: none; }
|
||||
|
||||
.menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; }
|
||||
.menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
|
||||
.menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
|
||||
.menu-checkbox-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
|
||||
input[type=checkbox] { visibility: hidden; }
|
||||
input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; }
|
||||
|
||||
|
|
@ -45,6 +48,7 @@ class Menu {
|
|||
this.menu = createElem(parent);
|
||||
this._id = 0;
|
||||
this._maxFPS = 0;
|
||||
this.hidden = 0;
|
||||
}
|
||||
|
||||
get newID() {
|
||||
|
|
@ -64,9 +68,22 @@ class Menu {
|
|||
return this.menu.offsetHeight;
|
||||
}
|
||||
|
||||
async addTitle(title) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'menu-title';
|
||||
el.id = this.newID;
|
||||
el.innerHTML = title;
|
||||
this.menu.appendChild(el);
|
||||
el.addEventListener('click', () => {
|
||||
this.hidden = !this.hidden;
|
||||
const all = document.getElementsByClassName('menu');
|
||||
for (const item of all) item.style.display = this.hidden ? 'none' : 'flex';
|
||||
});
|
||||
}
|
||||
|
||||
async addLabel(title) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'menu menu-title';
|
||||
el.className = 'menu menu-label';
|
||||
el.id = this.newID;
|
||||
el.innerHTML = title;
|
||||
this.menu.appendChild(el);
|
||||
|
|
@ -75,9 +92,9 @@ class Menu {
|
|||
async addBool(title, object, variable, callback) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'menu';
|
||||
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-label" for="${this.ID}"></label></div>${title}`;
|
||||
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`;
|
||||
this.menu.appendChild(el);
|
||||
document.getElementById(this.ID).addEventListener('change', (evt) => {
|
||||
el.addEventListener('change', (evt) => {
|
||||
object[variable] = evt.target.checked;
|
||||
if (callback) callback(evt.target.checked);
|
||||
});
|
||||
|
|
@ -88,7 +105,7 @@ class Menu {
|
|||
el.className = 'menu';
|
||||
el.innerHTML = `<input class="menu-range" type="range" id="${this.newID}" min="${min}" max="${max}" step="${step}" value="${object[variable]}">${title}`;
|
||||
this.menu.appendChild(el);
|
||||
document.getElementById(this.ID).addEventListener('change', (evt) => {
|
||||
el.addEventListener('change', (evt) => {
|
||||
object[variable] = evt.target.value;
|
||||
evt.target.setAttribute('value', evt.target.value);
|
||||
if (callback) callback(evt.target.value);
|
||||
|
|
@ -106,11 +123,14 @@ class Menu {
|
|||
async addButton(titleOn, titleOff, callback) {
|
||||
const el = document.createElement('button');
|
||||
el.className = 'menu menu-button';
|
||||
el.style.fontFamily = document.body.style.fontFamily;
|
||||
el.style.fontSize = document.body.style.fontSize;
|
||||
el.style.fontVariant = document.body.style.fontVariant;
|
||||
el.type = 'button';
|
||||
el.id = this.newID;
|
||||
el.innerText = titleOn;
|
||||
this.menu.appendChild(el);
|
||||
document.getElementById(this.ID).addEventListener('click', () => {
|
||||
el.addEventListener('click', () => {
|
||||
if (el.innerText === titleOn) el.innerText = titleOff;
|
||||
else el.innerText = titleOn;
|
||||
if (callback) callback(el.innerText !== titleOn);
|
||||
|
|
|
|||
20
demo/node.js
20
demo/node.js
|
|
@ -27,21 +27,15 @@ const config = {
|
|||
backend: 'tensorflow',
|
||||
console: true,
|
||||
face: {
|
||||
enabled: false,
|
||||
detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true, modelPath: 'file://models/facemesh/model.json', inputSize: 192 },
|
||||
iris: { enabled: true, modelPath: 'file://models/iris/model.json', inputSize: 192 },
|
||||
age: { enabled: true, modelPath: 'file://models/ssrnet-age/imdb/model.json', inputSize: 64, skipFrames: 5 },
|
||||
gender: { enabled: true, modelPath: 'file://models/ssrnet-gender/imdb/model.json' },
|
||||
detector: { modelPath: 'file://models/blazeface/back/model.json' },
|
||||
mesh: { modelPath: 'file://models/facemesh/model.json' },
|
||||
iris: { modelPath: 'file://models/iris/model.json' },
|
||||
age: { modelPath: 'file://models/ssrnet-age/imdb/model.json' },
|
||||
gender: { modelPath: 'file://models/ssrnet-gender/imdb/model.json' },
|
||||
emotion: { modelPath: 'file://models/emotion/model.json' },
|
||||
},
|
||||
body: { enabled: true, modelPath: 'file://models/posenet/model.json', inputResolution: 257, outputStride: 16, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
|
||||
body: { modelPath: 'file://models/posenet/model.json' },
|
||||
hand: {
|
||||
enabled: false,
|
||||
inputSize: 256,
|
||||
skipFrames: 10,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.3,
|
||||
scoreThreshold: 0.75,
|
||||
detector: { anchors: 'file://models/handdetect/anchors.json', modelPath: 'file://models/handdetect/model.json' },
|
||||
skeleton: { modelPath: 'file://models/handskeleton/model.json' },
|
||||
},
|
||||
|
|
|
|||
10
package.json
10
package.json
|
|
@ -37,12 +37,12 @@
|
|||
"rimraf": "^3.0.2"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node --trace-warnings --trace-uncaught --no-deprecation demo/node.js",
|
||||
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation demo/node.js",
|
||||
"lint": "eslint src/*.js demo/*.js",
|
||||
"build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/index.js",
|
||||
"build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/index.js",
|
||||
"build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/index.js",
|
||||
"build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/index.js",
|
||||
"build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --metafile=dist/human.json --outfile=dist/human.js src/human.js",
|
||||
"build-esm-bundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js",
|
||||
"build-esm-nobundle": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js",
|
||||
"build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --metafile=dist/human.cjs.json --outfile=dist/human.cjs src/human.js",
|
||||
"build": "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && ls -l dist/",
|
||||
"update": "npm update --depth 20 && npm dedupe && npm prune && npm audit",
|
||||
"changelog": "node changelog.js"
|
||||
|
|
|
|||
|
|
@ -22,11 +22,11 @@ async function load(config) {
|
|||
}
|
||||
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
if (frame >= config.face.emotion.skipFrames) {
|
||||
frame = 0;
|
||||
if (frame < config.face.emotion.skipFrames) {
|
||||
frame += 1;
|
||||
return last;
|
||||
}
|
||||
frame = 0;
|
||||
const enhance = tf.tidy(() => {
|
||||
if (image instanceof tf.Tensor) {
|
||||
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ class HandPose {
|
|||
}
|
||||
|
||||
async estimateHands(input, config) {
|
||||
this.maxContinuousChecks = config.skipFrames;
|
||||
this.skipFrames = config.skipFrames;
|
||||
this.detectionConfidence = config.minConfidence;
|
||||
this.maxHands = config.maxHands;
|
||||
const image = tf.tidy(() => {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ const models = {
|
|||
emotion: null,
|
||||
};
|
||||
|
||||
const override = {
|
||||
face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } },
|
||||
hand: { skipFrames: 0 },
|
||||
};
|
||||
|
||||
// helper function: gets elapsed time on both browser and nodejs
|
||||
const now = () => {
|
||||
if (typeof performance !== 'undefined') return performance.now();
|
||||
|
|
@ -66,9 +71,16 @@ function mergeDeep(...objects) {
|
|||
|
||||
function sanity(input) {
|
||||
if (!input) return 'input is not defined';
|
||||
const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
|
||||
if (!width || (width === 0)) return 'input is empty';
|
||||
if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
|
||||
if (tf.ENV.flags.IS_BROWSER && (input instanceof ImageData || input instanceof HTMLImageElement || input instanceof HTMLCanvasElement || input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) {
|
||||
const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
|
||||
if (!width || (width === 0)) return 'input is empty';
|
||||
}
|
||||
if (tf.ENV.flags.IS_BROWSER && (input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) {
|
||||
if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
|
||||
}
|
||||
if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
|
||||
return 'input must be a tensor';
|
||||
}
|
||||
try {
|
||||
tf.getBackend();
|
||||
} catch {
|
||||
|
|
@ -93,7 +105,8 @@ async function detect(input, userConfig = {}) {
|
|||
let timeStamp;
|
||||
|
||||
timeStamp = now();
|
||||
config = mergeDeep(defaults, userConfig);
|
||||
const shouldOverride = tf.ENV.flags.IS_NODE || (tf.ENV.flags.IS_BROWSER && !((input instanceof HTMLVideoElement) || (input instanceof HTMLMediaElement)));
|
||||
config = mergeDeep(defaults, userConfig, shouldOverride ? override : {});
|
||||
perf.config = Math.trunc(now() - timeStamp);
|
||||
|
||||
// sanity checks
|
||||
|
|
@ -222,3 +235,5 @@ exports.handpose = handpose;
|
|||
exports.tf = tf;
|
||||
exports.version = app.version;
|
||||
exports.state = state;
|
||||
|
||||
// Error: Failed to compile fragment shader
|
||||
Loading…
Reference in New Issue