fix for face detector when mesh is disabled

pull/91/head
Vladimir Mandic 2021-03-09 13:15:40 -05:00
parent 7dfdc16614
commit 57d4b767d1
23 changed files with 771 additions and 223 deletions

545
CHANGELOG.md Normal file
View File

@ -0,0 +1,545 @@
# @vladmandic/human
Version: **1.0.0**
Description: **Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
License: **MIT** </LICENSE>
Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/03/08 mandic00@live.com
- optimize for npm
### **0.40.9** 2021/03/08 mandic00@live.com
- fix performance issue when running with low confidence
### **0.40.8** 2021/03/08 mandic00@live.com
### **0.40.7** 2021/03/06 mandic00@live.com
- implemented 3d face angle calculations
### **0.40.6** 2021/03/06 mandic00@live.com
- add curve draw output
### **0.40.5** 2021/03/05 mandic00@live.com
- fix human.draw
### **0.40.4** 2021/03/05 mandic00@live.com
- cleanup blazepose code
- fix demo
### **0.40.3** 2021/03/05 mandic00@live.com
### **0.40.2** 2021/03/05 mandic00@live.com
- added blazepose-upper
### **0.40.1** 2021/03/04 mandic00@live.com
- implement blazepose and update demos
- add todo list
### **0.30.6** 2021/03/03 mandic00@live.com
- fine tuning age and face models
### **0.30.5** 2021/03/02 mandic00@live.com
- add debug logging flag
### **0.30.4** 2021/03/01 mandic00@live.com
- added skipinitial flag
### **0.30.3** 2021/02/28 mandic00@live.com
- typo
### **0.30.2** 2021/02/26 mandic00@live.com
- rebuild
- fix typo
### **0.30.1** 2021/02/25 mandic00@live.com
### **0.20.11** 2021/02/24 mandic00@live.com
### **0.20.10** 2021/02/22 mandic00@live.com
### **0.20.9** 2021/02/21 mandic00@live.com
- remove extra items
- simmilarity fix
### **0.20.8** 2021/02/21 mandic00@live.com
- embedding fix
### **0.20.7** 2021/02/21 mandic00@live.com
- build fix
### **0.20.6** 2021/02/21 mandic00@live.com
- embedding fix
### **0.20.5** 2021/02/21 mandic00@live.com
- fix imagefx and add dev builds
### **0.20.4** 2021/02/19 mandic00@live.com
- 0.20.4
### **0.20.3** 2021/02/17 mandic00@live.com
- rebuild
### **0.20.2** 2021/02/13 mandic00@live.com
- merge branch 'main' of https://github.com/vladmandic/human into main
- create codeql-analysis.yml
- create security.md
- add templates
### **0.20.1** 2021/02/08 mandic00@live.com
- menu fixes
- convert to typescript
### **0.11.5** 2021/02/06 mandic00@live.com
- added faceboxes alternative model
### **0.11.4** 2021/02/06 mandic00@live.com
### **0.11.3** 2021/02/02 mandic00@live.com
### **0.11.2** 2021/01/30 mandic00@live.com
- added warmup for nodejs
### **update for tfjs 3.0.0** 2021/01/29 mandic00@live.com
### **0.11.1** 2021/01/29 mandic00@live.com
### **0.10.2** 2021/01/22 mandic00@live.com
### **0.10.1** 2021/01/20 mandic00@live.com
### **0.9.26** 2021/01/18 mandic00@live.com
- fix face detection when mesh is disabled
- added minification notes
- version bump
### **0.9.25** 2021/01/13 mandic00@live.com
- added humangl custom backend
- rebuild
- code cleanup and enable minification
- fix safari incopatibility
### **0.9.24** 2021/01/12 mandic00@live.com
- work on blazepose
- full rebuild
### **0.9.23** 2021/01/11 mandic00@live.com
- added iris gesture
- fix emotion labels
- full rebuild
### **0.9.22** 2021/01/05 mandic00@live.com
- remove iris coords if iris is disabled
- web worker fix
### **0.9.21** 2021/01/03 mandic00@live.com
### **0.9.20** 2021/01/03 mandic00@live.com
- stricter linting, fix face annotations
### **0.9.19** 2020/12/23 mandic00@live.com
- added rawbox and rawmesh
- variable name changes, setting .rawcoords only if necessary
- option to return raw data (mesh, box) for facemesh / "preserve aspect ratio" fix from facemesh upstream
### **0.9.18** 2020/12/16 mandic00@live.com
- add z axis scaling
- major work on body module
- republish due to tfjs 2.8.0 issues
### **0.9.17** 2020/12/15 mandic00@live.com
- added custom webgl backend
### **0.9.16** 2020/12/12 mandic00@live.com
- change default ports
### **0.9.15** 2020/12/11 mandic00@live.com
- improved caching and warmup
- rebuild
### **0.9.14** 2020/12/10 mandic00@live.com
- conditional hand rotation
- staggered skipframes
- fix permissions
### **0.9.13** 2020/12/08 mandic00@live.com
- implemented face and hand boundary checks
- embedded sample for warmup
- switch to central logger
### **0.9.12** 2020/11/26 mandic00@live.com
- minor compatibility fixes
### **0.9.11** 2020/11/23 mandic00@live.com
- implement multi-person gestures
- modularize pipeline models
### **0.9.10** 2020/11/21 mandic00@live.com
- changed build for optimized node & browser
### **0.9.9** 2020/11/21 mandic00@live.com
- new screenshots
- camera exception handling
### **0.9.8** 2020/11/19 mandic00@live.com
- force f16 textures
- bugfix embedding check
### **0.9.7** 2020/11/19 mandic00@live.com
- ui redesign
### **0.9.6** 2020/11/18 mandic00@live.com
- optimize camera resize on mobile
- completed tfjs wrapper
### **0.9.5** 2020/11/17 mandic00@live.com
- fix serious performance bug around skipframes
- swtich to custom tfjs bundle
### **0.9.4** 2020/11/17 mandic00@live.com
- swtich to tfjs source import
### **0.9.3** 2020/11/16 mandic00@live.com
- switched to minified build
### **release: 1.2** 2020/11/15 mandic00@live.com
- full rebuild
### **0.9.2** 2020/11/14 mandic00@live.com
- fix camera restart on resize
### **0.9.1** 2020/11/13 mandic00@live.com
- version bump
- full rebuild
- implemented face embedding
- added internal benchmark tool
### **0.8.8** 2020/11/12 mandic00@live.com
- reduced bundle size
- implemented buffered processing
- fix for conditional model loading
### **0.8.7** 2020/11/11 mandic00@live.com
- added performance notes
- added notes on models
- fix bug in async ops and change imports
- fix wiki links
### **0.8.6** 2020/11/09 mandic00@live.com
- add wasm bundle
### **0.8.5** 2020/11/09 mandic00@live.com
- reimplemented blazeface processing
### **0.8.4** 2020/11/09 mandic00@live.com
- added additional gestures
- implemented blink detection
- fix wasm module
### **0.8.3** 2020/11/08 mandic00@live.com
- refresh
- optimizations
### **0.8.2** 2020/11/08 mandic00@live.com
- fix typos
- commit
### **0.8.1** 2020/11/07 mandic00@live.com
- fix hand detection performance
- optimized model loader
- merge branch 'main' of https://github.com/vladmandic/human into main
- created wiki
- delete bug_report.md
- optimize font resizing
- fix nms sync call
### **0.7.6** 2020/11/06 mandic00@live.com
- fixed memory leaks and updated docs
- model tuning
- cache invalidation improvements
- full async operations
### **0.7.5** 2020/11/05 mandic00@live.com
- implemented dev-server
### **0.7.4** 2020/11/05 mandic00@live.com
- fix canvas size on different orientation
- switched from es2020 to es2018 build target
### **0.7.3** 2020/11/05 mandic00@live.com
- optimized camera and mobile layout
- fixed worker and filter compatibility
- removed test code
### **0.7.2** 2020/11/04 mandic00@live.com
- major work on handpose model
### **0.7.1** 2020/11/04 mandic00@live.com
- changed demo build process
### **0.6.7** 2020/11/04 mandic00@live.com
- implemented simple gesture recognition
### **0.6.6** 2020/11/04 mandic00@live.com
- remove debug code
### **0.6.5** 2020/11/04 mandic00@live.com
- redo hand detection
### **0.6.4** 2020/11/03 mandic00@live.com
- added manifest
### **0.6.3** 2020/11/03 mandic00@live.com
- enhanced processing resolution
- fix pause restart
- complete model refactoring
- fixed typo
### **0.6.2** 2020/11/02 mandic00@live.com
- optimized demo
- package update
### **0.6.1** 2020/11/02 mandic00@live.com
- major performance improvements for all models
- revert "optimized canvas handling"
- optimized canvas handling
- minor optimization to imagefx
- fix demo image sample
- added tfjs-vis to distribution
### **0.5.5** 2020/11/01 mandic00@live.com
- changed defaults
### **0.5.4** 2020/11/01 mandic00@live.com
- implemented memory profiler
### **0.5.3** 2020/10/30 mandic00@live.com
- improved debug logging
### **0.5.2** 2020/10/30 mandic00@live.com
- added wasm and webgpu backends
### **0.5.1** 2020/10/30 mandic00@live.com
- improve demo line continous draws
### **0.4.10** 2020/10/30 mandic00@live.com
- fix for seedrandom
### **0.4.9** 2020/10/29 mandic00@live.com
### **0.4.8** 2020/10/28 mandic00@live.com
- revert "updated menu handler"
- fix webpack compatibility issue
### **0.4.7** 2020/10/27 mandic00@live.com
### **0.4.6** 2020/10/27 mandic00@live.com
- fix firefox compatibility bug
### **0.4.5** 2020/10/27 mandic00@live.com
### **0.4.4** 2020/10/27 mandic00@live.com
- implelented input resizing
### **0.4.3** 2020/10/22 mandic00@live.com
### **0.4.2** 2020/10/20 mandic00@live.com
- log initialization
### **0.4.1** 2020/10/19 mandic00@live.com
- breaking change: convert to object class
- compatibility notes
### **0.3.9** 2020/10/18 mandic00@live.com
- implemented image filters
- pure tensor pipeline without image converts
- autodetect skipframes
### **0.3.8** 2020/10/17 mandic00@live.com
- new menu layout
### **0.3.7** 2020/10/17 mandic00@live.com
- added diagnostics output
- parallelized agegender operations
### **0.3.6** 2020/10/17 mandic00@live.com
- fixed webcam initialization
- fixed memory leaks and added scoped runs
- modularized draw
- added state handling
- refactored package file layout
### **0.3.5** 2020/10/16 mandic00@live.com
- added auto-generated changelog
### **0.3.4** 2020/10/16 mandic00@live.com
- added camera selection
- optimized blazeface anchors
- added error handling
### **0.3.3** 2020/10/15 mandic00@live.com
- added blazeface back and front models
### **0.3.2** 2020/10/15 mandic00@live.com
- reduced web worker latency
- added debugging and versioning
- optimized demos and added scoped runs
- added multi backend support
### **0.3.1** 2020/10/14 mandic00@live.com
### **0.2.10** 2020/10/14 mandic00@live.com
- added emotion backend
- module parametrization and performance monitoring
- implemented multi-hand support
- fixed documentation typos
### **0.2.9** 2020/10/13 mandic00@live.com
- added node build and demo
### **0.2.8** 2020/10/13 mandic00@live.com
- added example image
### **0.2.7** 2020/10/13 mandic00@live.com
- new examples
### **0.2.6** 2020/10/13 mandic00@live.com
- enable all models by default
### **0.2.5** 2020/10/12 mandic00@live.com
- fixed memory leak
### **0.2.4** 2020/10/12 mandic00@live.com
- removed extra files
### **0.2.3** 2020/10/12 mandic00@live.com
### **0.2.2** 2020/10/12 mandic00@live.com
### **release: 1.0** 2020/10/12 mandic00@live.com
### **0.2.1** 2020/10/12 mandic00@live.com
- added sample image
- initial public commit
- initial commit

View File

@ -6,11 +6,11 @@ const userConfig = { backend: 'webgl' }; // add any user configuration overrides
/*
const userConfig = {
backend: 'webgl',
backend: 'wasm',
async: false,
warmup: 'face',
videoOptimized: true,
face: { enabled: true, iris: { enabled: false }, mesh: { enabled: true }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
videoOptimized: false,
face: { enabled: true, mesh: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, emotion: { enabled: false }, embedding: { enabled: false } },
hand: { enabled: false },
gesture: { enabled: false },
body: { enabled: false, modelType: 'blazepose', modelPath: '../models/blazepose.json' },

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,7 +1,7 @@
{
"inputs": {
"dist/human.esm.js": {
"bytes": 1353178,
"bytes": 1353275,
"imports": []
},
"demo/menu.js": {
@ -13,7 +13,7 @@
"imports": []
},
"demo/browser.js": {
"bytes": 28043,
"bytes": 28044,
"imports": [
{
"path": "dist/human.esm.js",
@ -35,7 +35,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 2061556
"bytes": 2061685
},
"dist/demo-browser-index.js": {
"imports": [],
@ -43,7 +43,7 @@
"entryPoint": "demo/browser.js",
"inputs": {
"dist/human.esm.js": {
"bytesInOutput": 1345684
"bytesInOutput": 1345779
},
"demo/menu.js": {
"bytesInOutput": 10696
@ -55,7 +55,7 @@
"bytesInOutput": 17538
}
},
"bytes": 1388062
"bytes": 1388157
}
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

58
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

20
dist/human.esm.json vendored
View File

@ -26,7 +26,7 @@
]
},
"src/blazeface/blazeface.ts": {
"bytes": 5706,
"bytes": 5704,
"imports": [
{
"path": "src/log.ts",
@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14034,
"bytes": 14105,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -446,11 +446,11 @@
"imports": []
},
"package.json": {
"bytes": 2559,
"bytes": 2604,
"imports": []
},
"src/draw.ts": {
"bytes": 16392,
"bytes": 16408,
"imports": [
{
"path": "config.js",
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 1979712
"bytes": 1979841
},
"dist/human.esm.js": {
"imports": [],
@ -581,7 +581,7 @@
"bytesInOutput": 1053
},
"src/blazeface/blazeface.ts": {
"bytesInOutput": 2194
"bytesInOutput": 2192
},
"src/blazeface/box.ts": {
"bytesInOutput": 834
@ -593,7 +593,7 @@
"bytesInOutput": 28983
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5112
"bytesInOutput": 5160
},
"src/human.ts": {
"bytesInOutput": 11955
@ -683,13 +683,13 @@
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2541
"bytesInOutput": 2578
},
"src/draw.ts": {
"bytesInOutput": 9740
"bytesInOutput": 9754
}
},
"bytes": 1353178
"bytes": 1353275
}
}
}

20
dist/human.iife.json vendored
View File

@ -26,7 +26,7 @@
]
},
"src/blazeface/blazeface.ts": {
"bytes": 5706,
"bytes": 5704,
"imports": [
{
"path": "src/log.ts",
@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14034,
"bytes": 14105,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -446,11 +446,11 @@
"imports": []
},
"package.json": {
"bytes": 2559,
"bytes": 2604,
"imports": []
},
"src/draw.ts": {
"bytes": 16392,
"bytes": 16408,
"imports": [
{
"path": "config.js",
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 1979723
"bytes": 1979852
},
"dist/human.ts": {
"imports": [],
@ -582,7 +582,7 @@
"bytesInOutput": 1053
},
"src/blazeface/blazeface.ts": {
"bytesInOutput": 2194
"bytesInOutput": 2192
},
"src/blazeface/box.ts": {
"bytesInOutput": 834
@ -594,7 +594,7 @@
"bytesInOutput": 28983
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5112
"bytesInOutput": 5160
},
"src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1535
@ -681,13 +681,13 @@
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2541
"bytesInOutput": 2578
},
"src/draw.ts": {
"bytesInOutput": 9740
"bytesInOutput": 9754
}
},
"bytes": 1353220
"bytes": 1353317
}
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

4
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

20
dist/human.node.json vendored
View File

@ -26,7 +26,7 @@
]
},
"src/blazeface/blazeface.ts": {
"bytes": 5706,
"bytes": 5704,
"imports": [
{
"path": "src/log.ts",
@ -56,7 +56,7 @@
"imports": []
},
"src/blazeface/facepipeline.ts": {
"bytes": 14034,
"bytes": 14105,
"imports": [
{
"path": "dist/tfjs.esm.js",
@ -446,11 +446,11 @@
"imports": []
},
"package.json": {
"bytes": 2559,
"bytes": 2604,
"imports": []
},
"src/draw.ts": {
"bytes": 16392,
"bytes": 16408,
"imports": [
{
"path": "config.js",
@ -553,7 +553,7 @@
"imports": [],
"exports": [],
"inputs": {},
"bytes": 746468
"bytes": 746597
},
"dist/human.node-gpu.js": {
"imports": [],
@ -582,10 +582,10 @@
"bytesInOutput": 1145
},
"src/blazeface/blazeface.ts": {
"bytesInOutput": 2338
"bytesInOutput": 2333
},
"src/blazeface/facepipeline.ts": {
"bytesInOutput": 5159
"bytesInOutput": 5207
},
"src/blazeface/box.ts": {
"bytesInOutput": 854
@ -681,13 +681,13 @@
"bytesInOutput": 55295
},
"package.json": {
"bytesInOutput": 2538
"bytesInOutput": 2575
},
"src/draw.ts": {
"bytesInOutput": 9629
"bytesInOutput": 9643
}
},
"bytes": 290596
"bytes": 290690
}
}
}

58
dist/human.ts vendored

File diff suppressed because one or more lines are too long

4
dist/human.ts.map vendored

File diff suppressed because one or more lines are too long

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@vladmandic/human",
"version": "0.40.9",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@vladmandic/human",
"version": "0.40.9",
"version": "1.0.0",
"license": "MIT",
"devDependencies": {
"@tensorflow/tfjs": "^3.2.0",

View File

@ -78,7 +78,7 @@ export class BlazeFaceModel {
const [detectedOutputs, boxes, scores] = tf.tidy(() => {
const resizedImage = inputImage.resizeBilinear([this.width, this.height]);
// const normalizedImage = tf.mul(tf.sub(resizedImage.div(255), 0.5), 2);
const normalizedImage = tf.sub(resizedImage.div(127.5), 1);
const normalizedImage = resizedImage.div(127.5).sub(0.5);
const batchedPrediction = this.blazeFaceModel.predict(normalizedImage);
let prediction;
// are we using tfhub or pinto converted model?

View File

@ -30,7 +30,7 @@ export function cutBoxFromImageAndResize(box, image, cropSize) {
return tf.image.cropAndResize(image, boxes, [0], cropSize);
}
export function enlargeBox(box, factor = 1.6) {
export function enlargeBox(box, factor = 1.5) {
const center = getBoxCenter(box);
const size = getBoxSize(box);
const newHalfSize = [factor * size[0] / 2, factor * size[1] / 2];

View File

@ -185,7 +185,7 @@ export class Pipeline {
let face;
let angle = 0;
let rotationMatrix;
if (config.face.detector.rotation) {
if (config.face.detector.rotation && config.face.mesh.enabled) {
const [indexOfMouth, indexOfForehead] = (box.landmarks.length >= LANDMARKS_COUNT) ? MESH_KEYPOINTS_LINE_OF_SYMMETRY_INDICES : BLAZEFACE_KEYPOINTS_LINE_OF_SYMMETRY_INDICES;
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
@ -205,6 +205,7 @@ export class Pipeline {
coords: null,
box,
faceConfidence: null,
boxConfidence: box.confidence,
confidence: box.confidence,
image: face,
};
@ -258,10 +259,12 @@ export class Pipeline {
return prediction;
}));
results = results.filter((a) => a !== null);
// remove cache entries for detected boxes on low confidence
if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config.face.detector.minConfidence);
this.detectedFaces = results.length;
return results;
}

View File

@ -130,7 +130,7 @@ export async function face(inCanvas, result) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' '));
}
if (f.angle) labels.push(`roll: ${Math.trunc(100 * f.angle.roll) / 100} yaw:${Math.trunc(100 * f.angle.yaw) / 100} pitch:${Math.trunc(100 * f.angle.pitch) / 100}`);
if (f.angle && f.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.angle.roll) / 100} yaw:${Math.trunc(100 * f.angle.yaw) / 100} pitch:${Math.trunc(100 * f.angle.pitch) / 100}`);
if (labels.length === 0) labels.push('face');
ctx.fillStyle = options.color;
for (let i = labels.length - 1; i >= 0; i--) {