ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switched face embedding to mobileface

pull/280/head
Vladimir Mandic 2021-03-12 12:54:08 -05:00
parent 162ace9fc3
commit 12b0058a1b
10 changed files with 90 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human # @vladmandic/human
Version: **1.0.3** Version: **1.1.0**
Description: **Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition** Description: **Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>** Author: **Vladimir Mandic <mandic00@live.com>**
@ -11,14 +11,16 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/03/11 mandic00@live.com ### **HEAD -> main** 2021/03/11 mandic00@live.com
### **1.0.4** 2021/03/11 mandic00@live.com
- add face return tensor
- add test for face descriptors
- wip on embedding - wip on embedding
- simplify face box coordinate calculations - simplify face box coordinate calculations
- annotated models and removed gender-ssrnet - annotated models and removed gender-ssrnet
- autodetect inputsizes - autodetect inputsizes
### **origin/main** 2021/03/10 mandic00@live.com
### **1.0.3** 2021/03/10 mandic00@live.com ### **1.0.3** 2021/03/10 mandic00@live.com
- strong typing for public classes and hide private classes - strong typing for public classes and hide private classes

2
README.md
View File

@ -69,7 +69,7 @@ Default models in Human library are:
- **Gender Detection**: Oarriaga Gender - **Gender Detection**: Oarriaga Gender
- **Age Detection**: SSR-Net Age IMDB - **Age Detection**: SSR-Net Age IMDB
- **Body Analysis**: PoseNet - **Body Analysis**: PoseNet
- **Face Embedding**: Sirius-AI MobileFaceNet Embedding - **Face Embedding**: BecauseofAI MobileFace Embedding
Note that alternative models are provided and can be enabled via configuration Note that alternative models are provided and can be enabled via configuration
For example, `PoseNet` model can be switched for `BlazePose` model depending on the use case For example, `PoseNet` model can be switched for `BlazePose` model depending on the use case

10
TODO.md
View File

@ -8,3 +8,13 @@
- Explore EfficientPose - Explore EfficientPose
<https://github.com/daniegr/EfficientPose> <https://github.com/daniegr/EfficientPose>
<https://github.com/PINTO0309/PINTO_model_zoo/tree/main/084_EfficientPose> <https://github.com/PINTO0309/PINTO_model_zoo/tree/main/084_EfficientPose>
## WiP: Embedding
- Implement offsetRaw
full with and without rotation
full with and without embedding
full with any without mesh
embedding with and without mesh
boxRaw and meshRaw with and without mesh

6
config.js
View File

@ -121,9 +121,9 @@ export default {
}, },
embedding: { embedding: {
enabled: false, // to improve accuracy of face embedding extraction it is recommended enabled: false, // to improve accuracy of face embedding extraction it is
// to enable detector.rotation and mesh.enabled // highly recommended to enable detector.rotation and mesh.enabled
modelPath: '../models/mobilefacenet.json', modelPath: '../models/mobileface.json',
}, },
}, },

13
demo/browser.js
View File

@ -84,11 +84,22 @@ let original;
async function calcSimmilariry(result) { async function calcSimmilariry(result) {
document.getElementById('compare-container').style.display = human.config.face.embedding.enabled ? 'block' : 'none'; document.getElementById('compare-container').style.display = human.config.face.embedding.enabled ? 'block' : 'none';
if (!human.config.face.embedding.enabled) return; if (!human.config.face.embedding.enabled) return;
if (!(result?.face?.length > 0) || (result?.face[0]?.embedding?.length !== 192)) return; if (!(result?.face?.length > 0) || (result?.face[0]?.embedding?.length !== 256)) return;
if (!original) { if (!original) {
original = result; original = result;
if (result.face[0].tensor) {
const enhanced = human.enhance(result.face[0]);
if (enhanced) {
const c = document.getElementById('orig');
const squeeze = enhanced.squeeze();
human.tf.browser.toPixels(squeeze, c);
enhanced.dispose();
squeeze.dispose();
}
} else {
document.getElementById('compare-canvas').getContext('2d').drawImage(original.canvas, 0, 0, 200, 200); document.getElementById('compare-canvas').getContext('2d').drawImage(original.canvas, 0, 0, 200, 200);
} }
}
const simmilarity = human.simmilarity(original?.face[0]?.embedding, result?.face[0]?.embedding); const simmilarity = human.simmilarity(original?.face[0]?.embedding, result?.face[0]?.embedding);
document.getElementById('simmilarity').innerText = `simmilarity: ${Math.trunc(1000 * simmilarity) / 10}%`; document.getElementById('simmilarity').innerText = `simmilarity: ${Math.trunc(1000 * simmilarity) / 10}%`;
} }

4
demo/embedding.html
View File

@ -26,9 +26,9 @@
<body> <body>
<br>Sample Images: <br>Sample Images:
<div id="images"></div> <div id="images"></div>
<br>Selected Face<br> <br>Selected Face (Enhanced)<br>
<canvas id="orig" style="width: 200px; height: 200px;"></canvas> <canvas id="orig" style="width: 200px; height: 200px;"></canvas>
<br>Extracted Faces - click on a face to sort by simmilarity:<br> <br><br>Extracted Faces - click on a face to sort by simmilarity:<br>
<div id="faces"></div> <div id="faces"></div>
</body> </body>
</html> </html>

39
demo/embedding.js
View File

@ -11,7 +11,7 @@ const userConfig = {
enabled: true, enabled: true,
detector: { rotation: true, return: true }, detector: { rotation: true, return: true },
mesh: { enabled: true }, mesh: { enabled: true },
embedding: { enabled: true, modelPath: '../models/mobilefacenet.json' }, embedding: { enabled: true },
iris: { enabled: false }, iris: { enabled: false },
age: { enabled: false }, age: { enabled: false },
gender: { enabled: false }, gender: { enabled: false },
@ -21,12 +21,15 @@ const userConfig = {
gesture: { enabled: false }, gesture: { enabled: false },
body: { enabled: false }, body: { enabled: false },
}; };
const human = new Human(userConfig);
const human = new Human(userConfig); // new instance of human
const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg']; const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'];
// const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg', // const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg',
// '../private/me (1).jpg', '../private/me (2).jpg', '../private/me (3).jpg', '../private/me (4).jpg', '../private/me (5).jpg', '../private/me (6).jpg', '../private/me (7).jpg', '../private/me (8).jpg', // '../private/me (1).jpg', '../private/me (2).jpg', '../private/me (3).jpg', '../private/me (4).jpg', '../private/me (5).jpg', '../private/me (6).jpg', '../private/me (7).jpg', '../private/me (8).jpg',
// '../private/me (9).jpg', '../private/me (10).jpg', '../private/me (11).jpg', '../private/me (12).jpg', '../private/me (13).jpg']; // '../private/me (9).jpg', '../private/me (10).jpg', '../private/me (11).jpg', '../private/me (12).jpg', '../private/me (13).jpg'];
const all = [];
const all = []; // array that will hold all detected faces
function log(...msg) { function log(...msg) {
const dt = new Date(); const dt = new Date();
@ -38,14 +41,24 @@ function log(...msg) {
async function analyze(face) { async function analyze(face) {
log('Face:', face); log('Face:', face);
const box = [[0.05, 0.15, 0.90, 0.85]]; // top, left, bottom, right // if we have face image tensor, enhance it and display it
const crop = human.tf.image.cropAndResize(face.tensor.expandDims(0), box, [0], [200, 200]); // optionally do a tight box crop if (face.tensor) {
const enhanced = human.enhance(face);
if (enhanced) {
const c = document.getElementById('orig'); const c = document.getElementById('orig');
human.tf.browser.toPixels(crop.squeeze(), c); const squeeze = enhanced.squeeze();
human.tf.browser.toPixels(squeeze, c);
enhanced.dispose();
squeeze.dispose();
}
}
// loop through all canvases that contain faces
const canvases = document.getElementsByClassName('face'); const canvases = document.getElementsByClassName('face');
for (const canvas of canvases) { for (const canvas of canvases) {
// calculate simmilarity from selected face to current one in the loop
const res = human.simmilarity(face.embedding, all[canvas.tag.sample][canvas.tag.face].embedding); const res = human.simmilarity(face.embedding, all[canvas.tag.sample][canvas.tag.face].embedding);
// draw the canvas and simmilarity score
canvas.title = res; canvas.title = res;
await human.tf.browser.toPixels(all[canvas.tag.sample][canvas.tag.face].tensor, canvas); await human.tf.browser.toPixels(all[canvas.tag.sample][canvas.tag.face].tensor, canvas);
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
@ -55,6 +68,8 @@ async function analyze(face) {
ctx.fillStyle = 'rgba(255, 255, 255, 1)'; ctx.fillStyle = 'rgba(255, 255, 255, 1)';
ctx.fillText(`${(100 * res).toFixed(1)}%`, 4, 20); ctx.fillText(`${(100 * res).toFixed(1)}%`, 4, 20);
} }
// sort all faces by simmilarity
const sorted = document.getElementById('faces'); const sorted = document.getElementById('faces');
[...sorted.children] [...sorted.children]
.sort((a, b) => parseFloat(b.title) - parseFloat(a.title)) .sort((a, b) => parseFloat(b.title) - parseFloat(a.title))
@ -70,22 +85,26 @@ async function faces(index, res) {
canvas.width = 200; canvas.width = 200;
canvas.height = 200; canvas.height = 200;
canvas.className = 'face'; canvas.className = 'face';
// mouse click on any face canvas triggers analysis
canvas.addEventListener('click', (evt) => { canvas.addEventListener('click', (evt) => {
log('Select:', 'Image:', evt.target.tag.sample, 'Face:', evt.target.tag.face); log('Select:', 'Image:', evt.target.tag.sample, 'Face:', evt.target.tag.face);
analyze(all[evt.target.tag.sample][evt.target.tag.face]); analyze(all[evt.target.tag.sample][evt.target.tag.face]);
}); });
// if we actually got face image tensor, draw canvas with that face
if (res.face[i].tensor) {
human.tf.browser.toPixels(res.face[i].tensor, canvas); human.tf.browser.toPixels(res.face[i].tensor, canvas);
document.getElementById('faces').appendChild(canvas); document.getElementById('faces').appendChild(canvas);
} }
}
} }
async function add(index) { async function add(index) {
log('Add image:', samples[index]); log('Add image:', samples[index]);
return new Promise((resolve) => { return new Promise((resolve) => {
const img = new Image(100, 100); const img = new Image(100, 100);
img.onload = () => { img.onload = () => { // must wait until image is loaded
human.detect(img).then((res) => faces(index, res)); human.detect(img).then((res) => faces(index, res)); // then wait until image is analyzed
document.getElementById('images').appendChild(img); document.getElementById('images').appendChild(img); // and finally we can add it
resolve(true); resolve(true);
}; };
img.title = samples[index]; img.title = samples[index];
@ -95,7 +114,7 @@ async function add(index) {
async function main() { async function main() {
await human.load(); await human.load();
for (const i in samples) await add(i); for (const i in samples) await add(i); // download and analyze all images
log('Ready'); log('Ready');
} }

11
demo/node.js
View File

@ -1,8 +1,10 @@
const log = require('@vladmandic/pilogger'); const log = require('@vladmandic/pilogger');
const fs = require('fs'); const fs = require('fs');
const process = require('process'); const process = require('process');
// for Node, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu'); const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
// load specific version of Human library that matches TensorFlow mode // load specific version of Human library that matches TensorFlow mode
const Human = require('../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default; const Human = require('../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default;
@ -15,15 +17,16 @@ const myConfig = {
async: false, async: false,
face: { face: {
enabled: true, enabled: true,
detector: { modelPath: 'file://models/blazeface-back.json', enabled: true }, detector: { modelPath: 'file://models/blazeface-back.json', enabled: true, rotation: false },
mesh: { modelPath: 'file://models/facemesh.json', enabled: true }, mesh: { modelPath: 'file://models/facemesh.json', enabled: true },
iris: { modelPath: 'file://models/iris.json', enabled: true }, iris: { modelPath: 'file://models/iris.json', enabled: true },
age: { modelPath: 'file://models/age.json', enabled: true }, age: { modelPath: 'file://models/age.json', enabled: true },
gender: { modelPath: 'file://models/gender.json', enabled: true }, gender: { modelPath: 'file://models/gender.json', enabled: true },
emotion: { modelPath: 'file://models/emotion.json', enabled: true }, emotion: { modelPath: 'file://models/emotion.json', enabled: true },
embedding: { modelPath: 'file://models/mobileface.json', enabled: true },
}, },
// body: { modelPath: 'file://models/blazepose.json', modelType: 'blazepose', enabled: true }, // body: { modelPath: 'file://models/blazepose.json', enabled: true },
body: { modelPath: 'file://models/posenet.json', modelType: 'posenet', enabled: true }, body: { modelPath: 'file://models/posenet.json', enabled: true },
hand: { hand: {
enabled: true, enabled: true,
detector: { modelPath: 'file://models/handdetect.json' }, detector: { modelPath: 'file://models/handdetect.json' },

6
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "1.0.4", "version": "1.1.0",
"description": "Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition", "description": "Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",
@ -54,13 +54,13 @@
"@tensorflow/tfjs-layers": "^3.3.0", "@tensorflow/tfjs-layers": "^3.3.0",
"@tensorflow/tfjs-node": "^3.3.0", "@tensorflow/tfjs-node": "^3.3.0",
"@tensorflow/tfjs-node-gpu": "^3.3.0", "@tensorflow/tfjs-node-gpu": "^3.3.0",
"@types/node": "^14.14.33", "@types/node": "^14.14.34",
"@typescript-eslint/eslint-plugin": "^4.17.0", "@typescript-eslint/eslint-plugin": "^4.17.0",
"@typescript-eslint/parser": "^4.17.0", "@typescript-eslint/parser": "^4.17.0",
"@vladmandic/pilogger": "^0.2.14", "@vladmandic/pilogger": "^0.2.14",
"chokidar": "^3.5.1", "chokidar": "^3.5.1",
"dayjs": "^1.10.4", "dayjs": "^1.10.4",
"esbuild": "^0.9.0", "esbuild": "=0.9.0",
"eslint": "^7.21.0", "eslint": "^7.21.0",
"eslint-config-airbnb-base": "^14.2.1", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1", "eslint-plugin-import": "^2.22.1",

13
src/human.ts
View File

@ -151,6 +151,11 @@ class Human {
return 0; return 0;
} }
enhance(input: any): any {
if (this.config.face.embedding.enabled) return embedding.enhance(input);
return null;
}
// preload models, not explicitly required as it's done automatically on first use // preload models, not explicitly required as it's done automatically on first use
async load(userConfig = null) { async load(userConfig = null) {
this.state = 'load'; this.state = 'load';
@ -359,11 +364,11 @@ class Human {
// run emotion, inherits face from blazeface // run emotion, inherits face from blazeface
this.#analyze('Start Embedding:'); this.#analyze('Start Embedding:');
if (this.config.async) { if (this.config.async) {
embeddingRes = this.config.face.embedding.enabled ? embedding.predict(face.image, this.config) : []; embeddingRes = this.config.face.embedding.enabled ? embedding.predict(face, this.config) : [];
} else { } else {
this.state = 'run:embedding'; this.state = 'run:embedding';
timeStamp = now(); timeStamp = now();
embeddingRes = this.config.face.embedding.enabled ? await embedding.predict(face.image, this.config) : []; embeddingRes = this.config.face.embedding.enabled ? await embedding.predict(face, this.config) : [];
this.#perf.embedding = Math.trunc(now() - timeStamp); this.#perf.embedding = Math.trunc(now() - timeStamp);
} }
this.#analyze('End Emotion:'); this.#analyze('End Emotion:');
@ -388,6 +393,8 @@ class Human {
// combine results // combine results
faceRes.push({ faceRes.push({
...face,
/*
confidence: face.confidence, confidence: face.confidence,
faceConfidence: face.faceConfidence, faceConfidence: face.faceConfidence,
boxConfidence: face.boxConfidence, boxConfidence: face.boxConfidence,
@ -395,7 +402,9 @@ class Human {
mesh: face.mesh, mesh: face.mesh,
boxRaw: face.boxRaw, boxRaw: face.boxRaw,
meshRaw: face.meshRaw, meshRaw: face.meshRaw,
offsetRaw: face.offsetRaw,
annotations: face.annotations, annotations: face.annotations,
*/
age: ageRes.age, age: ageRes.age,
gender: genderRes.gender, gender: genderRes.gender,
genderConfidence: genderRes.confidence, genderConfidence: genderRes.confidence,