mirror of https://github.com/vladmandic/human
added emotion backend
Vladimir Mandic
parent
e1a514d7ca
commit
095e5be481
68
README.md
68
README.md
|
|
@ -1,4 +1,4 @@
|
|||
# Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking and Age & Gender Prediction
|
||||
# Human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction & Emotion Prediction
|
||||
|
||||
- [**Documentation**](https://github.com/vladmandic/human#readme)
|
||||
- [**Code Repository**](https://github.com/vladmandic/human)
|
||||
|
|
@ -22,19 +22,6 @@ Compatible with Browser, WebWorker and NodeJS execution!
|
|||
|
||||
<hr>
|
||||
|
||||
## Credits
|
||||
|
||||
This is an amalgamation of multiple existing models:
|
||||
|
||||
- Face Detection: [**MediaPipe BlazeFace**](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view)
|
||||
- Facial Spacial Geometry: [**MediaPipe FaceMesh**](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view)
|
||||
- Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view)
|
||||
- Hand Detection & Skeleton: [**MediaPipe HandPose**](https://drive.google.com/file/d/1sv4sSb9BSNVZhLzxXJ0jBv9DqD-4jnAz/view)
|
||||
- Body Pose Detection: [**PoseNet**](https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5)
|
||||
- Age & Gender Prediction: [**SSR-Net**](https://github.com/shamangary/SSR-Net)
|
||||
|
||||
<hr>
|
||||
|
||||
## Installation
|
||||
|
||||
**Important**
|
||||
|
|
@ -198,7 +185,7 @@ human.defaults = {
|
|||
detector: {
|
||||
modelPath: '../models/blazeface/model.json',
|
||||
maxFaces: 10,
|
||||
skipFrames: 5,
|
||||
skipFrames: 10,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.3,
|
||||
scoreThreshold: 0.75,
|
||||
|
|
@ -214,12 +201,19 @@ human.defaults = {
|
|||
age: {
|
||||
enabled: true,
|
||||
modelPath: '../models/ssrnet-imdb-age/model.json',
|
||||
skipFrames: 5,
|
||||
skipFrames: 10,
|
||||
},
|
||||
gender: {
|
||||
enabled: true,
|
||||
modelPath: '../models/ssrnet-imdb-gender/model.json',
|
||||
},
|
||||
emotion: {
|
||||
enabled: true,
|
||||
minConfidence: 0.5,
|
||||
skipFrames: 10,
|
||||
useGrayscale: true,
|
||||
modelPath: '../models/emotion/model.json',
|
||||
},
|
||||
},
|
||||
body: {
|
||||
enabled: true,
|
||||
|
|
@ -230,7 +224,7 @@ human.defaults = {
|
|||
},
|
||||
hand: {
|
||||
enabled: true,
|
||||
skipFrames: 5,
|
||||
skipFrames: 10,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.3,
|
||||
scoreThreshold: 0.75,
|
||||
|
|
@ -253,6 +247,7 @@ Where:
|
|||
- `minConfidence`: threshold for discarding a prediction
|
||||
- `iouThreshold`: threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
- `scoreThreshold`: threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
- `useGrayscale`: convert color input to grayscale before processing or use single channels when color input is not supported
|
||||
- `nmsRadius`: radius for deciding points are too close in non-maximum suppression
|
||||
|
||||
<hr>
|
||||
|
|
@ -268,18 +263,18 @@ result = {
|
|||
{
|
||||
confidence, // <number>
|
||||
box, // <array [x, y, width, height]>
|
||||
mesh, // <array of 3D points [x, y, z]> (468 base points & 10 iris points)
|
||||
annotations, // <list of object { landmark: array of points }> (32 base annotated landmarks & 2 iris annotations)
|
||||
iris, // <number> (relative distance of iris to camera, multiple by focal lenght to get actual distance)
|
||||
age, // <number> (estimated age)
|
||||
gender, // <string> (male or female)
|
||||
mesh, // <array of 3D points [x, y, z]> 468 base points & 10 iris points
|
||||
annotations, // <list of object { landmark: array of points }> 32 base annotated landmarks & 2 iris annotations
|
||||
iris, // <number> relative distance of iris to camera, multiple by focal lenght to get actual distance
|
||||
age, // <number> estimated age
|
||||
gender, // <string> 'male', 'female'
|
||||
}
|
||||
],
|
||||
body: // <array of detected objects>
|
||||
[
|
||||
{
|
||||
score, // <number>,
|
||||
keypoints, // <array of 2D landmarks [ score, landmark, position [x, y] ]> (17 annotated landmarks)
|
||||
keypoints, // <array of 2D landmarks [ score, landmark, position [x, y] ]> 17 annotated landmarks
|
||||
}
|
||||
],
|
||||
hand: // <array of detected objects>
|
||||
|
|
@ -287,8 +282,15 @@ result = {
|
|||
{
|
||||
confidence, // <number>,
|
||||
box, // <array [x, y, width, height]>,
|
||||
landmarks, // <array of 3D points [x, y,z]> (21 points)
|
||||
annotations, // <array of 3D landmarks [ landmark: <array of points> ]> (5 annotated landmakrs)
|
||||
landmarks, // <array of 3D points [x, y,z]> 21 points
|
||||
annotations, // <array of 3D landmarks [ landmark: <array of points> ]> 5 annotated landmakrs
|
||||
}
|
||||
],
|
||||
emotion: // <array of emotions>
|
||||
[
|
||||
{
|
||||
score, // <number> probabily of emotion
|
||||
emotion, // <string> 'angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral'
|
||||
}
|
||||
],
|
||||
}
|
||||
|
|
@ -302,6 +304,7 @@ Additionally, `result` object includes internal performance data - total time sp
|
|||
hand,
|
||||
face,
|
||||
agegender,
|
||||
emotion,
|
||||
total,
|
||||
}
|
||||
```
|
||||
|
|
@ -343,6 +346,7 @@ Performance per module:
|
|||
- Face Iris: 25 FPS (includes face detect and face geometry)
|
||||
- Age: 60 FPS (includes face detect)
|
||||
- Gender: 60 FPS (includes face detect)
|
||||
- Emotion: 60 FPS (includes face detect)
|
||||
- Hand: 40 FPS
|
||||
- Body: 50 FPS
|
||||
|
||||
|
|
@ -350,7 +354,19 @@ Library can also be used on mobile devices
|
|||
|
||||
<hr>
|
||||
|
||||
## Credits
|
||||
|
||||
- Face Detection: [**MediaPipe BlazeFace**](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view)
|
||||
- Facial Spacial Geometry: [**MediaPipe FaceMesh**](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view)
|
||||
- Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view)
|
||||
- Hand Detection & Skeleton: [**MediaPipe HandPose**](https://drive.google.com/file/d/1sv4sSb9BSNVZhLzxXJ0jBv9DqD-4jnAz/view)
|
||||
- Body Pose Detection: [**PoseNet**](https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5)
|
||||
- Age & Gender Prediction: [**SSR-Net**](https://github.com/shamangary/SSR-Net)
|
||||
- Emotion Prediction: [**Oarriaga**](https://github.com/oarriaga/face_classification)
|
||||
|
||||
<hr>
|
||||
|
||||
## Todo
|
||||
|
||||
- Tweak default parameters
|
||||
- Tweak default parameters and factorization for age/gender/emotion
|
||||
- Verify age/gender models
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
<script src="https://cdn.jsdelivr.net/npm/quicksettings"></script>
|
||||
<script src="./demo-esm.js" type="module"></script>
|
||||
</head>
|
||||
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'">
|
||||
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-variant: small-caps">
|
||||
<div id="main">
|
||||
<video id="video" playsinline style="display: none"></video>
|
||||
<image id="image" src="" style="display: none"></video>
|
||||
<canvas id="canvas"></canvas>
|
||||
<div id="log">Starting Human library</div>
|
||||
<div id="log">Human library</div>
|
||||
</div>
|
||||
</body>
|
||||
|
|
|
|||
|
|
@ -4,36 +4,52 @@ import human from '../dist/human.esm.js';
|
|||
|
||||
const ui = {
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseLabel: 'rgba(255, 200, 255, 0.8)',
|
||||
baseFont: 'small-caps 1.2rem "Segoe UI"',
|
||||
baseLineWidth: 16,
|
||||
};
|
||||
|
||||
const config = {
|
||||
face: {
|
||||
enabled: false,
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 10 },
|
||||
gender: { enabled: true },
|
||||
emotion: { enabled: true, minConfidence: 0.5, useGrayscale: true },
|
||||
},
|
||||
body: { enabled: false, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
};
|
||||
let settings;
|
||||
|
||||
function str(...msg) {
|
||||
if (!Array.isArray(msg)) return msg;
|
||||
let line = '';
|
||||
for (const entry of msg) {
|
||||
if (typeof entry === 'object') line += JSON.stringify(entry).replace(/{|}|"|\[|\]/g, '').replace(/,/g, ', ');
|
||||
else line += entry;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
async function drawFace(result, canvas) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.strokeStyle = ui.baseColor;
|
||||
ctx.font = ui.baseFont;
|
||||
for (const face of result) {
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
|
||||
}
|
||||
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
|
||||
const labelAgeGender = `${face.gender || ''} ${face.age || ''}`;
|
||||
const labelIris = face.iris ? `iris: ${face.iris}` : '';
|
||||
const labelEmotion = face.emotion && face.emotion[0] ? `emotion: ${Math.trunc(100 * face.emotion[0].score)}% ${face.emotion[0].emotion}` : '';
|
||||
ctx.fillStyle = ui.baseLabel;
|
||||
ctx.fillText(`face ${labelAgeGender} ${labelIris} ${labelEmotion}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
|
||||
ctx.stroke();
|
||||
ctx.lineWidth = 1;
|
||||
if (face.mesh) {
|
||||
|
|
@ -140,7 +156,9 @@ async function drawHand(result, canvas) {
|
|||
if (settings.getValue('Draw Boxes')) {
|
||||
ctx.lineWidth = ui.baseLineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.fillStyle = ui.baseColor;
|
||||
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
|
||||
ctx.fillStyle = ui.baseLabel;
|
||||
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
|
@ -199,11 +217,10 @@ async function runHumanDetect(input, canvas) {
|
|||
drawHand(result.hand, canvas);
|
||||
// update log
|
||||
const engine = await human.tf.engine();
|
||||
const memory = `Memory: ${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
|
||||
log.innerText = `
|
||||
TFJS Version: ${human.tf.version_core} Memory: ${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors
|
||||
GPU Memory: used ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes free ${Math.floor(1024 * 1024 * engine.backendInstance.numMBBeforeWarning).toLocaleString()} bytes
|
||||
Result Object Size: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes
|
||||
Performance: ${JSON.stringify(result.performance)}
|
||||
TFJS Version: ${human.tf.version_core} | ${memory} | GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes
|
||||
Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
|
||||
`;
|
||||
// rinse & repeate
|
||||
// if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes
|
||||
|
|
@ -214,28 +231,36 @@ async function runHumanDetect(input, canvas) {
|
|||
function setupGUI() {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
settings.addBoolean('Pause', false, (val) => {
|
||||
const style = document.createElement('style');
|
||||
// style.type = 'text/css';
|
||||
style.innerHTML = `
|
||||
.qs_main { font: 1rem "Segoe UI"; }
|
||||
.qs_label { font: 0.8rem "Segoe UI"; }
|
||||
.qs_title_bar { display: none; }
|
||||
.qs_content { background: darkslategray; }
|
||||
.qs_container { background: transparent; color: white; margin: 6px; padding: 6px; }
|
||||
.qs_checkbox_label { top: 2px; }
|
||||
.qs_button { width: -webkit-fill-available; font: 1rem "Segoe UI"; cursor: pointer; }
|
||||
`;
|
||||
document.getElementsByTagName('head')[0].appendChild(style);
|
||||
settings.addButton('Play/Pause', () => {
|
||||
const video = document.getElementById('video');
|
||||
const canvas = document.getElementById('canvas');
|
||||
if (val) video.pause();
|
||||
else video.play();
|
||||
if (!video.paused) {
|
||||
document.getElementById('log').innerText = 'Paused ...';
|
||||
video.pause();
|
||||
} else {
|
||||
document.getElementById('log').innerText = 'Starting Human Library ...';
|
||||
video.play();
|
||||
}
|
||||
runHumanDetect(video, canvas);
|
||||
});
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addBoolean('Draw Boxes', false);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
settings.addBoolean('Fill Polygons', true);
|
||||
settings.bindText('baseColor', ui.baseColor, config);
|
||||
settings.bindText('baseFont', ui.baseFont, config);
|
||||
settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config);
|
||||
settings.addHTML('line2', '<hr>'); settings.hideTitle('line2');
|
||||
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
|
||||
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
|
||||
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
|
||||
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
|
||||
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
|
||||
settings.addBoolean('Face Emotion', config.face.emotion.enabled, (val) => config.face.emotion.enabled = val);
|
||||
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
|
||||
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
|
||||
settings.addHTML('line3', '<hr>'); settings.hideTitle('line3');
|
||||
|
|
@ -245,11 +270,13 @@ function setupGUI() {
|
|||
});
|
||||
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
|
||||
config.face.detector.skipFrames = parseInt(val);
|
||||
config.face.emotion.skipFrames = parseInt(val);
|
||||
config.face.age.skipFrames = parseInt(val);
|
||||
config.hand.skipFrames = parseInt(val);
|
||||
});
|
||||
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
|
||||
config.face.detector.minConfidence = parseFloat(val);
|
||||
config.face.emotion.minConfidence = parseFloat(val);
|
||||
config.hand.minConfidence = parseFloat(val);
|
||||
});
|
||||
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
|
||||
|
|
@ -261,6 +288,13 @@ function setupGUI() {
|
|||
config.face.detector.iouThreshold = parseFloat(val);
|
||||
config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addBoolean('Draw Boxes', true);
|
||||
settings.addBoolean('Draw Points', true);
|
||||
settings.addBoolean('Draw Polygons', true);
|
||||
settings.addBoolean('Fill Polygons', true);
|
||||
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
|
||||
settings.addRange('FPS', 0, 100, 0, 1);
|
||||
}
|
||||
|
||||
async function setupCanvas(input) {
|
||||
|
|
@ -289,6 +323,7 @@ async function setupCamera() {
|
|||
video.width = video.videoWidth;
|
||||
video.height = video.videoHeight;
|
||||
video.play();
|
||||
video.pause();
|
||||
resolve(video);
|
||||
};
|
||||
});
|
||||
|
|
@ -316,9 +351,9 @@ async function main() {
|
|||
// or setup image
|
||||
// const image = await setupImage();
|
||||
// setup output canvas from input object, select video or image
|
||||
const canvas = await setupCanvas(video);
|
||||
await setupCanvas(video);
|
||||
// run actual detection. if input is video, it will run in a loop else it will run only once
|
||||
runHumanDetect(video, canvas);
|
||||
// runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
window.onload = main;
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@ const ui = {
|
|||
const config = {
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 5 },
|
||||
gender: { enabled: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
};
|
||||
let settings;
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ const logger = new console.Console({
|
|||
const config = {
|
||||
face: {
|
||||
enabled: false,
|
||||
detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true, modelPath: 'file://models/facemesh/model.json', inputSize: 192 },
|
||||
iris: { enabled: true, modelPath: 'file://models/iris/model.json', inputSize: 192 },
|
||||
age: { enabled: true, modelPath: 'file://models/ssrnet-age/imdb/model.json', inputSize: 64, skipFrames: 5 },
|
||||
|
|
@ -36,7 +36,7 @@ const config = {
|
|||
hand: {
|
||||
enabled: false,
|
||||
inputSize: 256,
|
||||
skipFrames: 5,
|
||||
skipFrames: 10,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.3,
|
||||
scoreThreshold: 0.75,
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ const ui = {
|
|||
const config = {
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
age: { enabled: true, skipFrames: 5 },
|
||||
gender: { enabled: true },
|
||||
},
|
||||
body: { enabled: true, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
|
||||
};
|
||||
let settings;
|
||||
let worker;
|
||||
|
|
|
|||
|
|
@ -69201,6 +69201,67 @@ var require_ssrnet = __commonJS((exports) => {
|
|||
exports.loadGender = loadGender;
|
||||
});
|
||||
|
||||
// src/emotion/emotion.js
|
||||
var require_emotion = __commonJS((exports) => {
|
||||
const tf = require_tf_node();
|
||||
const annotations = ["angry", "discust", "fear", "happy", "sad", "surpise", "neutral"];
|
||||
const models = {};
|
||||
let last = [];
|
||||
let frame = 0;
|
||||
const multiplier = 1.5;
|
||||
function getImage(image, size) {
|
||||
const tensor = tf.tidy(() => {
|
||||
const buffer = tf.browser.fromPixels(image, 1);
|
||||
const resize = tf.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf.cast(tf.expandDims(resize, 0), "float32");
|
||||
return expand;
|
||||
});
|
||||
return tensor;
|
||||
}
|
||||
async function load(config) {
|
||||
if (!models.emotion)
|
||||
models.emotion = await tf.loadGraphModel(config.face.emotion.modelPath);
|
||||
}
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
if (frame >= config.face.emotion.skipFrames) {
|
||||
frame = 0;
|
||||
return last;
|
||||
}
|
||||
const enhance = tf.tidy(() => {
|
||||
if (image instanceof tf.Tensor) {
|
||||
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
|
||||
const [r, g, b] = tf.split(resize, 3, 3);
|
||||
if (config.face.emotion.useGrayscale) {
|
||||
const r1 = tf.mul(r, [0.2989]);
|
||||
const g1 = tf.mul(g, [0.587]);
|
||||
const b1 = tf.mul(b, [0.114]);
|
||||
const grayscale = tf.addN([r1, g1, b1]);
|
||||
return grayscale;
|
||||
}
|
||||
return g;
|
||||
}
|
||||
return getImage(image, config.face.emotion.inputSize);
|
||||
});
|
||||
const obj = [];
|
||||
if (config.face.emotion.enabled) {
|
||||
const emotionT = await models.emotion.predict(enhance);
|
||||
const data = await emotionT.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (multiplier * data[i] > config.face.emotion.minConfidence)
|
||||
obj.push({score: Math.min(0.99, Math.trunc(100 * multiplier * data[i]) / 100), emotion: annotations[i]});
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf.dispose(emotionT);
|
||||
}
|
||||
tf.dispose(enhance);
|
||||
last = obj;
|
||||
return obj;
|
||||
}
|
||||
exports.predict = predict;
|
||||
exports.load = load;
|
||||
});
|
||||
|
||||
// src/posenet/modelBase.js
|
||||
var require_modelBase = __commonJS((exports) => {
|
||||
const tf = require_tf_node();
|
||||
|
|
@ -70294,6 +70355,14 @@ var require_config = __commonJS((exports) => {
|
|||
gender: {
|
||||
enabled: true,
|
||||
modelPath: "../models/ssrnet-gender/imdb/model.json"
|
||||
},
|
||||
emotion: {
|
||||
enabled: true,
|
||||
inputSize: 64,
|
||||
minConfidence: 0.5,
|
||||
skipFrames: 10,
|
||||
useGrayscale: true,
|
||||
modelPath: "../models/emotion/model.json"
|
||||
}
|
||||
},
|
||||
body: {
|
||||
|
|
@ -70330,6 +70399,7 @@ var require_src = __commonJS((exports) => {
|
|||
const tf = require_tf_node();
|
||||
const facemesh = require_facemesh();
|
||||
const ssrnet = require_ssrnet();
|
||||
const emotion = require_emotion();
|
||||
const posenet = require_posenet();
|
||||
const handpose = require_handpose();
|
||||
const defaults = require_config().default;
|
||||
|
|
@ -70363,6 +70433,8 @@ var require_src = __commonJS((exports) => {
|
|||
await ssrnet.loadAge(config);
|
||||
if (config.face.gender.enabled)
|
||||
await ssrnet.loadGender(config);
|
||||
if (config.face.emotion.enabled)
|
||||
await emotion.load(config);
|
||||
if (config.body.enabled && !models.posenet)
|
||||
models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose)
|
||||
|
|
@ -70396,6 +70468,9 @@ var require_src = __commonJS((exports) => {
|
|||
timeStamp = performance.now();
|
||||
const ssrdata = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
face.image.dispose();
|
||||
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
|
||||
faceRes.push({
|
||||
|
|
@ -70405,6 +70480,7 @@ var require_src = __commonJS((exports) => {
|
|||
annotations: face.annotations,
|
||||
age: ssrdata.age,
|
||||
gender: ssrdata.gender,
|
||||
emotion: emotiondata,
|
||||
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
|
||||
});
|
||||
}
|
||||
|
|
@ -70412,7 +70488,6 @@ var require_src = __commonJS((exports) => {
|
|||
tf.env().set("WEBGL_PACK_DEPTHWISECONV", savedWebglPackDepthwiseConvFlag);
|
||||
tf.engine().endScope();
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
console.log("total", perf.total);
|
||||
resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -3939,6 +3939,67 @@ var require_ssrnet = __commonJS((exports2) => {
|
|||
exports2.loadGender = loadGender;
|
||||
});
|
||||
|
||||
// src/emotion/emotion.js
|
||||
var require_emotion = __commonJS((exports2) => {
|
||||
const tf2 = require("@tensorflow/tfjs");
|
||||
const annotations = ["angry", "discust", "fear", "happy", "sad", "surpise", "neutral"];
|
||||
const models2 = {};
|
||||
let last = [];
|
||||
let frame = 0;
|
||||
const multiplier = 1.5;
|
||||
function getImage(image, size) {
|
||||
const tensor = tf2.tidy(() => {
|
||||
const buffer = tf2.browser.fromPixels(image, 1);
|
||||
const resize = tf2.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf2.cast(tf2.expandDims(resize, 0), "float32");
|
||||
return expand;
|
||||
});
|
||||
return tensor;
|
||||
}
|
||||
async function load(config) {
|
||||
if (!models2.emotion)
|
||||
models2.emotion = await tf2.loadGraphModel(config.face.emotion.modelPath);
|
||||
}
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
if (frame >= config.face.emotion.skipFrames) {
|
||||
frame = 0;
|
||||
return last;
|
||||
}
|
||||
const enhance = tf2.tidy(() => {
|
||||
if (image instanceof tf2.Tensor) {
|
||||
const resize = tf2.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
|
||||
const [r, g, b] = tf2.split(resize, 3, 3);
|
||||
if (config.face.emotion.useGrayscale) {
|
||||
const r1 = tf2.mul(r, [0.2989]);
|
||||
const g1 = tf2.mul(g, [0.587]);
|
||||
const b1 = tf2.mul(b, [0.114]);
|
||||
const grayscale = tf2.addN([r1, g1, b1]);
|
||||
return grayscale;
|
||||
}
|
||||
return g;
|
||||
}
|
||||
return getImage(image, config.face.emotion.inputSize);
|
||||
});
|
||||
const obj = [];
|
||||
if (config.face.emotion.enabled) {
|
||||
const emotionT = await models2.emotion.predict(enhance);
|
||||
const data = await emotionT.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (multiplier * data[i] > config.face.emotion.minConfidence)
|
||||
obj.push({score: Math.min(0.99, Math.trunc(100 * multiplier * data[i]) / 100), emotion: annotations[i]});
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf2.dispose(emotionT);
|
||||
}
|
||||
tf2.dispose(enhance);
|
||||
last = obj;
|
||||
return obj;
|
||||
}
|
||||
exports2.predict = predict;
|
||||
exports2.load = load;
|
||||
});
|
||||
|
||||
// src/posenet/modelBase.js
|
||||
var require_modelBase = __commonJS((exports2) => {
|
||||
const tf2 = require("@tensorflow/tfjs");
|
||||
|
|
@ -5032,6 +5093,14 @@ var require_config = __commonJS((exports2) => {
|
|||
gender: {
|
||||
enabled: true,
|
||||
modelPath: "../models/ssrnet-gender/imdb/model.json"
|
||||
},
|
||||
emotion: {
|
||||
enabled: true,
|
||||
inputSize: 64,
|
||||
minConfidence: 0.5,
|
||||
skipFrames: 10,
|
||||
useGrayscale: true,
|
||||
modelPath: "../models/emotion/model.json"
|
||||
}
|
||||
},
|
||||
body: {
|
||||
|
|
@ -5067,6 +5136,7 @@ var require_config = __commonJS((exports2) => {
|
|||
const tf = require("@tensorflow/tfjs");
|
||||
const facemesh = require_facemesh();
|
||||
const ssrnet = require_ssrnet();
|
||||
const emotion = require_emotion();
|
||||
const posenet = require_posenet();
|
||||
const handpose = require_handpose();
|
||||
const defaults = require_config().default;
|
||||
|
|
@ -5100,6 +5170,8 @@ async function detect(input, userConfig) {
|
|||
await ssrnet.loadAge(config);
|
||||
if (config.face.gender.enabled)
|
||||
await ssrnet.loadGender(config);
|
||||
if (config.face.emotion.enabled)
|
||||
await emotion.load(config);
|
||||
if (config.body.enabled && !models.posenet)
|
||||
models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose)
|
||||
|
|
@ -5133,6 +5205,9 @@ async function detect(input, userConfig) {
|
|||
timeStamp = performance.now();
|
||||
const ssrdata = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
timeStamp = performance.now();
|
||||
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
face.image.dispose();
|
||||
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
|
||||
faceRes.push({
|
||||
|
|
@ -5142,6 +5217,7 @@ async function detect(input, userConfig) {
|
|||
annotations: face.annotations,
|
||||
age: ssrdata.age,
|
||||
gender: ssrdata.gender,
|
||||
emotion: emotiondata,
|
||||
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
|
||||
});
|
||||
}
|
||||
|
|
@ -5149,7 +5225,6 @@ async function detect(input, userConfig) {
|
|||
tf.env().set("WEBGL_PACK_DEPTHWISECONV", savedWebglPackDepthwiseConvFlag);
|
||||
tf.engine().endScope();
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
console.log("total", perf.total);
|
||||
resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -31,6 +31,14 @@ export default {
|
|||
enabled: true,
|
||||
modelPath: '../models/ssrnet-gender/imdb/model.json',
|
||||
},
|
||||
emotion: {
|
||||
enabled: true,
|
||||
inputSize: 64, // fixed value
|
||||
minConfidence: 0.5,
|
||||
skipFrames: 10,
|
||||
useGrayscale: true,
|
||||
modelPath: '../models/emotion/model.json',
|
||||
},
|
||||
},
|
||||
body: {
|
||||
enabled: true,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
const tf = require('@tensorflow/tfjs');
|
||||
|
||||
const annotations = ['angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral'];
|
||||
const models = {};
|
||||
let last = [];
|
||||
let frame = 0;
|
||||
const multiplier = 1.5;
|
||||
|
||||
function getImage(image, size) {
|
||||
const tensor = tf.tidy(() => {
|
||||
const buffer = tf.browser.fromPixels(image, 1);
|
||||
const resize = tf.image.resizeBilinear(buffer, [size, size]);
|
||||
const expand = tf.cast(tf.expandDims(resize, 0), 'float32');
|
||||
return expand;
|
||||
});
|
||||
return tensor;
|
||||
}
|
||||
|
||||
async function load(config) {
|
||||
if (!models.emotion) models.emotion = await tf.loadGraphModel(config.face.emotion.modelPath);
|
||||
}
|
||||
|
||||
async function predict(image, config) {
|
||||
frame += 1;
|
||||
if (frame >= config.face.emotion.skipFrames) {
|
||||
frame = 0;
|
||||
return last;
|
||||
}
|
||||
const enhance = tf.tidy(() => {
|
||||
if (image instanceof tf.Tensor) {
|
||||
const resize = tf.image.resizeBilinear(image, [config.face.emotion.inputSize, config.face.emotion.inputSize], false);
|
||||
const [r, g, b] = tf.split(resize, 3, 3);
|
||||
if (config.face.emotion.useGrayscale) {
|
||||
// 0.2989 * R + 0.5870 * G + 0.1140 * B // https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
const r1 = tf.mul(r, [0.2989]);
|
||||
const g1 = tf.mul(g, [0.5870]);
|
||||
const b1 = tf.mul(b, [0.1140]);
|
||||
const grayscale = tf.addN([r1, g1, b1]);
|
||||
return grayscale;
|
||||
}
|
||||
return g;
|
||||
}
|
||||
return getImage(image, config.face.emotion.inputSize);
|
||||
});
|
||||
const obj = [];
|
||||
if (config.face.emotion.enabled) {
|
||||
const emotionT = await models.emotion.predict(enhance);
|
||||
const data = await emotionT.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (multiplier * data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * multiplier * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf.dispose(emotionT);
|
||||
}
|
||||
tf.dispose(enhance);
|
||||
last = obj;
|
||||
return obj;
|
||||
}
|
||||
|
||||
exports.predict = predict;
|
||||
exports.load = load;
|
||||
11
src/index.js
11
src/index.js
|
|
@ -1,6 +1,7 @@
|
|||
const tf = require('@tensorflow/tfjs');
|
||||
const facemesh = require('./facemesh/facemesh.js');
|
||||
const ssrnet = require('./ssrnet/ssrnet.js');
|
||||
const emotion = require('./emotion/emotion.js');
|
||||
const posenet = require('./posenet/posenet.js');
|
||||
const handpose = require('./handpose/handpose.js');
|
||||
const defaults = require('./config.js').default;
|
||||
|
|
@ -38,6 +39,7 @@ async function detect(input, userConfig) {
|
|||
// load models if enabled
|
||||
if (config.face.age.enabled) await ssrnet.loadAge(config);
|
||||
if (config.face.gender.enabled) await ssrnet.loadGender(config);
|
||||
if (config.face.emotion.enabled) await emotion.load(config);
|
||||
if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand);
|
||||
if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face);
|
||||
|
|
@ -76,7 +78,12 @@ async function detect(input, userConfig) {
|
|||
timeStamp = performance.now();
|
||||
const ssrdata = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(performance.now() - timeStamp);
|
||||
// run emotion, inherits face from blazeface
|
||||
timeStamp = performance.now();
|
||||
const emotiondata = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(performance.now() - timeStamp);
|
||||
face.image.dispose();
|
||||
// calculate iris distance
|
||||
// iris: array[ bottom, left, top, right, center ]
|
||||
const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
|
||||
? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0])
|
||||
|
|
@ -88,7 +95,8 @@ async function detect(input, userConfig) {
|
|||
annotations: face.annotations,
|
||||
age: ssrdata.age,
|
||||
gender: ssrdata.gender,
|
||||
iris: (iris !== 0) ? Math.trunc(100 * 11.7 / iris) / 100 : 0,
|
||||
emotion: emotiondata,
|
||||
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -98,7 +106,6 @@ async function detect(input, userConfig) {
|
|||
tf.engine().endScope();
|
||||
// combine results
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
console.log('total', perf.total);
|
||||
resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf });
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue