ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implemented simple gesture recognition

pull/50/head
Vladimir Mandic 2020-11-04 10:18:22 -05:00
parent 42fdac3c28
commit 30c3a89c94
24 changed files with 1124 additions and 916 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -1,6 +1,6 @@
# Human Library # Human Library
## 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction & Emotion Prediction ## 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition
- [**Documentation**](https://github.com/vladmandic/human#readme) - [**Documentation**](https://github.com/vladmandic/human#readme)
- [**Code Repository**](https://github.com/vladmandic/human) - [**Code Repository**](https://github.com/vladmandic/human)
@ -361,6 +361,11 @@ config = {
modelPath: '../models/handskeleton/model.json', modelPath: '../models/handskeleton/model.json',
}, },
}, },
gesture: {
enabled: true, // enable simple gesture recognition
// takes processed data and based on geometry detects simple gestures
// easily expandable via code, see `src/gesture.js`
},
}; };
``` ```
@ -408,10 +413,17 @@ result = {
emotion, // <string> 'angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral' emotion, // <string> 'angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral'
} }
], ],
gesture: // object containing parsed gestures
{
face, // <array of string>
body, // <array of string>
hand, // <array of string>
}
performance = { // performance data of last execution for each module measuredin miliseconds performance = { // performance data of last execution for each module measuredin miliseconds
backend, // time to initialize tf backend, valid only during backend startup backend, // time to initialize tf backend, valid only during backend startup
load, // time to load models, valid only during model load load, // time to load models, valid only during model load
image, // time for image processing image, // time for image processing
gesture, // gesture analysis time
body, // model time body, // model time
hand, // model time hand, // model time
face, // model time face, // model time
@ -484,6 +496,7 @@ For example, it can perform multiple face detections at 60+ FPS, but drops to ~1
- Enabled all: 15 FPS - Enabled all: 15 FPS
- Image filters: 80 FPS (standalone) - Image filters: 80 FPS (standalone)
- Gesture: 80 FPS (standalone)
- Face Detect: 80 FPS (standalone) - Face Detect: 80 FPS (standalone)
- Face Geometry: 30 FPS (includes face detect) - Face Geometry: 30 FPS (includes face detect)
- Face Iris: 30 FPS (includes face detect and face geometry) - Face Iris: 30 FPS (includes face detect and face geometry)
@ -495,8 +508,9 @@ For example, it can perform multiple face detections at 60+ FPS, but drops to ~1
### Performance per module on a **smartphone** with Snapdragon 855 on a FullHD input: ### Performance per module on a **smartphone** with Snapdragon 855 on a FullHD input:
- Enabled all: 3 FPS - Enabled all: 5 FPS
- Image filters: 30 FPS (standalone) - Image filters: 30 FPS (standalone)
- Gesture: 30 FPS (standalone)
- Face Detect: 20 FPS (standalone) - Face Detect: 20 FPS (standalone)
- Face Geometry: 10 FPS (includes face detect) - Face Geometry: 10 FPS (includes face detect)
- Face Iris: 5 FPS (includes face detect and face geometry) - Face Iris: 5 FPS (includes face detect and face geometry)

3
config.js
View File

@ -41,6 +41,9 @@ export default {
polaroid: false, // image polaroid camera effect polaroid: false, // image polaroid camera effect
pixelate: 0, // range: 0 (no pixelate) to N (number of pixels to pixelate) pixelate: 0, // range: 0 (no pixelate) to N (number of pixels to pixelate)
}, },
gesture: {
enabled: true, // enable simple gesture recognition
},
face: { face: {
enabled: true, // controls if specified modul is enabled enabled: true, // controls if specified modul is enabled
// face.enabled is required for all face models: detector, mesh, iris, age, gender, emotion // face.enabled is required for all face models: detector, mesh, iris, age, gender, emotion

24
demo/browser.js
View File

@ -25,6 +25,8 @@ const ui = {
useDepth: true, useDepth: true,
console: true, console: true,
maxFrames: 10, maxFrames: 10,
modelsPreload: true,
modelsWarmup: true,
}; };
// configuration overrides // configuration overrides
@ -62,6 +64,7 @@ const config = {
}, },
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
gesture: { enabled: true },
}; };
// global variables // global variables
@ -123,15 +126,17 @@ function drawResults(input, result, canvas) {
draw.face(result.face, canvas, ui, human.facemesh.triangulation); draw.face(result.face, canvas, ui, human.facemesh.triangulation);
draw.body(result.body, canvas, ui); draw.body(result.body, canvas, ui);
draw.hand(result.hand, canvas, ui); draw.hand(result.hand, canvas, ui);
draw.gesture(result.gesture, canvas, ui);
// update log // update log
const engine = human.tf.engine(); const engine = human.tf.engine();
const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : ''; const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : '';
const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`; const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`;
const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : ''; const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : '';
const avg = Math.trunc(10 * fps.reduce((a, b) => a + b) / fps.length) / 10;
document.getElementById('log').innerText = ` document.getElementById('log').innerText = `
video: ${camera.name} | facing: ${camera.facing} | resolution: ${camera.width} x ${camera.height} ${processing} video: ${camera.name} | facing: ${camera.facing} | resolution: ${camera.width} x ${camera.height} ${processing}
backend: ${human.tf.getBackend()} | ${memory} | object size: ${(str(result)).length.toLocaleString()} bytes backend: ${human.tf.getBackend()} | ${memory} | object size: ${(str(result)).length.toLocaleString()} bytes
performance: ${str(result.performance)} performance: ${str(result.performance)} FPS:${avg}
`; `;
} }
@ -159,7 +164,13 @@ async function setupCamera() {
try { try {
stream = await navigator.mediaDevices.getUserMedia({ stream = await navigator.mediaDevices.getUserMedia({
audio: false, audio: false,
video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight, resizeMode: 'none' }, video: {
facingMode: (ui.facing ? 'user' : 'environment'),
width: window.innerWidth,
height: window.innerHeight,
resizeMode: 'none',
contrast: 75,
},
}); });
} catch (err) { } catch (err) {
output.innerText += '\nCamera permission denied'; output.innerText += '\nCamera permission denied';
@ -267,7 +278,7 @@ async function detectVideo() {
document.getElementById('canvas').style.display = 'block'; document.getElementById('canvas').style.display = 'block';
const video = document.getElementById('video'); const video = document.getElementById('video');
const canvas = document.getElementById('canvas'); const canvas = document.getElementById('canvas');
ui.baseFont = ui.baseFontProto.replace(/{size}/, '1.2rem'); ui.baseFont = ui.baseFontProto.replace(/{size}/, '1.3rem');
ui.baseLineHeight = ui.baseLineHeightProto; ui.baseLineHeight = ui.baseLineHeightProto;
if ((video.srcObject !== null) && !video.paused) { if ((video.srcObject !== null) && !video.paused) {
document.getElementById('play').style.display = 'block'; document.getElementById('play').style.display = 'block';
@ -286,7 +297,7 @@ async function detectVideo() {
async function detectSampleImages() { async function detectSampleImages() {
document.getElementById('play').style.display = 'none'; document.getElementById('play').style.display = 'none';
config.videoOptimized = false; config.videoOptimized = false;
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${1.2 * ui.columns}rem`); ui.baseFont = ui.baseFontProto.replace(/{size}/, `${1.3 * ui.columns}rem`);
ui.baseLineHeight = ui.baseLineHeightProto * ui.columns; ui.baseLineHeight = ui.baseLineHeightProto * ui.columns;
document.getElementById('canvas').style.display = 'none'; document.getElementById('canvas').style.display = 'none';
document.getElementById('samples-container').style.display = 'block'; document.getElementById('samples-container').style.display = 'block';
@ -318,6 +329,7 @@ function setupMenu() {
menu.addBool('Face Emotion', config.face.emotion, 'enabled'); menu.addBool('Face Emotion', config.face.emotion, 'enabled');
menu.addBool('Body Pose', config.body, 'enabled'); menu.addBool('Body Pose', config.body, 'enabled');
menu.addBool('Hand Pose', config.hand, 'enabled'); menu.addBool('Hand Pose', config.hand, 'enabled');
menu.addBool('Gesture Analysis', config.gesture, 'enabled');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">'); menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Model Parameters'); menu.addLabel('Model Parameters');
@ -383,11 +395,15 @@ async function main() {
setupMenu(); setupMenu();
document.getElementById('log').innerText = `Human: version ${human.version} TensorFlow/JS: version ${human.tf.version_core}`; document.getElementById('log').innerText = `Human: version ${human.version} TensorFlow/JS: version ${human.tf.version_core}`;
// this is not required, just pre-warms the library // this is not required, just pre-warms the library
if (ui.modelsPreload) {
status('loading'); status('loading');
await human.load(); await human.load();
}
if (ui.modelsWarmup) {
status('initializing'); status('initializing');
const warmup = new ImageData(50, 50); const warmup = new ImageData(50, 50);
await human.detect(warmup); await human.detect(warmup);
}
status('human: ready'); status('human: ready');
document.getElementById('loader').style.display = 'none'; document.getElementById('loader').style.display = 'none';
document.getElementById('play').style.display = 'block'; document.getElementById('play').style.display = 'block';

18
demo/draw.js
View File

@ -1,3 +1,18 @@
async function drawGesture(result, canvas, ui) {
if (!result) return;
const ctx = canvas.getContext('2d');
ctx.font = ui.baseFont;
ctx.fillStyle = ui.baseLabel;
let i = 1;
for (const [key, val] of Object.entries(result)) {
if (val.length > 0) {
const label = `${key}: ${val.join(', ')}`;
ctx.fillText(label, 6, i * (ui.baseLineHeight + 24));
i += 1;
}
}
}
async function drawFace(result, canvas, ui, triangulation) { async function drawFace(result, canvas, ui, triangulation) {
if (!result) return; if (!result) return;
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
@ -17,7 +32,7 @@ async function drawFace(result, canvas, ui, triangulation) {
if (face.iris) labels.push(`iris: ${face.iris}`); if (face.iris) labels.push(`iris: ${face.iris}`);
if (face.emotion && face.emotion[0]) labels.push(`${Math.trunc(100 * face.emotion[0].score)}% ${face.emotion[0].emotion}`); if (face.emotion && face.emotion[0]) labels.push(`${Math.trunc(100 * face.emotion[0].score)}% ${face.emotion[0].emotion}`);
ctx.fillStyle = ui.baseLabel; ctx.fillStyle = ui.baseLabel;
for (const i in labels) ctx.fillText(labels[i], face.box[0] + 6, face.box[1] + 24 + ((i + 1) * ui.baseLineHeight)); for (const i in labels) ctx.fillText(labels[i], face.box[0] + 8, face.box[1] + 24 + ((i + 1) * ui.baseLineHeight));
ctx.stroke(); ctx.stroke();
ctx.lineWidth = 1; ctx.lineWidth = 1;
if (face.mesh) { if (face.mesh) {
@ -171,6 +186,7 @@ const draw = {
face: drawFace, face: drawFace,
body: drawBody, body: drawBody,
hand: drawHand, hand: drawHand,
gesture: drawGesture,
}; };
export default draw; export default draw;

2
demo/index.html
View File

@ -27,7 +27,7 @@
.status { position: absolute; width: 100vw; top: 100px; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px darkslategrey; } .status { position: absolute; width: 100vw; top: 100px; text-align: center; font-size: 4rem; font-weight: 100; text-shadow: 2px 2px darkslategrey; }
.thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; } .thumbnail { margin: 8px; box-shadow: 0 0 4px 4px dimgrey; }
.thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); } .thumbnail:hover { box-shadow: 0 0 8px 8px dimgrey; filter: grayscale(1); }
.log { position: fixed; bottom: 0; } .log { position: fixed; bottom: 0; margin: 0.4rem; }
.samples-container { display: flex; flex-wrap: wrap; } .samples-container { display: flex; flex-wrap: wrap; }
.video { display: none; } .video { display: none; }
.canvas { margin: 0 auto; width: 100%; } .canvas { margin: 0 auto; width: 100%; }

14
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

53
dist/human.esm-nobundle.json vendored
View File

@ -1,11 +1,11 @@
{ {
"inputs": { "inputs": {
"config.js": { "config.js": {
"bytes": 7216, "bytes": 7300,
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2915, "bytes": 2989,
"imports": [] "imports": []
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
@ -74,6 +74,10 @@
"bytes": 19592, "bytes": 19592,
"imports": [] "imports": []
}, },
"src/gesture.js": {
"bytes": 2122,
"imports": []
},
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytes": 224151, "bytes": 224151,
"imports": [] "imports": []
@ -91,7 +95,7 @@
] ]
}, },
"src/handpose/handpipeline.js": { "src/handpose/handpipeline.js": {
"bytes": 7861, "bytes": 7959,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
@ -120,7 +124,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15209, "bytes": 11168,
"imports": [ "imports": [
{ {
"path": "src/facemesh/facemesh.js" "path": "src/facemesh/facemesh.js"
@ -138,7 +142,10 @@
"path": "src/handpose/handpose.js" "path": "src/handpose/handpose.js"
}, },
{ {
"path": "src/imagefx.js" "path": "src/gesture.js"
},
{
"path": "src/image.js"
}, },
{ {
"path": "src/profile.js" "path": "src/profile.js"
@ -151,6 +158,14 @@
} }
] ]
}, },
"src/image.js": {
"bytes": 4603,
"imports": [
{
"path": "src/imagefx.js"
}
]
},
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19352, "bytes": 19352,
"imports": [] "imports": []
@ -275,7 +290,7 @@
"dist/human.esm-nobundle.js.map": { "dist/human.esm-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 606968 "bytes": 610990
}, },
"dist/human.esm-nobundle.js": { "dist/human.esm-nobundle.js": {
"imports": [], "imports": [],
@ -311,7 +326,7 @@
"bytesInOutput": 1236 "bytesInOutput": 1236
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1098 "bytesInOutput": 1104
}, },
"src/posenet/modelBase.js": { "src/posenet/modelBase.js": {
"bytesInOutput": 455 "bytesInOutput": 455
@ -326,25 +341,25 @@
"bytesInOutput": 546 "bytesInOutput": 546
}, },
"src/posenet/keypoints.js": { "src/posenet/keypoints.js": {
"bytesInOutput": 1620 "bytesInOutput": 1621
}, },
"src/posenet/vectors.js": { "src/posenet/vectors.js": {
"bytesInOutput": 606 "bytesInOutput": 607
}, },
"src/posenet/decodePose.js": { "src/posenet/decodePose.js": {
"bytesInOutput": 1015 "bytesInOutput": 1016
}, },
"src/posenet/decodeMultiple.js": { "src/posenet/decodeMultiple.js": {
"bytesInOutput": 603 "bytesInOutput": 603
}, },
"src/posenet/util.js": { "src/posenet/util.js": {
"bytesInOutput": 1052 "bytesInOutput": 1053
}, },
"src/posenet/modelPoseNet.js": { "src/posenet/modelPoseNet.js": {
"bytesInOutput": 841 "bytesInOutput": 841
}, },
"src/posenet/posenet.js": { "src/posenet/posenet.js": {
"bytesInOutput": 458 "bytesInOutput": 459
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytesInOutput": 1420 "bytesInOutput": 1420
@ -364,23 +379,29 @@
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1112 "bytesInOutput": 1112
}, },
"src/gesture.js": {
"bytesInOutput": 1200
},
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11013 "bytesInOutput": 11013
}, },
"src/image.js": {
"bytesInOutput": 2379
},
"config.js": { "config.js": {
"bytesInOutput": 1272 "bytesInOutput": 1293
}, },
"package.json": { "package.json": {
"bytesInOutput": 2567 "bytesInOutput": 2641
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8467 "bytesInOutput": 5570
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 212078 "bytes": 212868
} }
} }
} }

492
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

44
dist/human.esm.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"config.js": { "config.js": {
"bytes": 7216, "bytes": 7300,
"imports": [] "imports": []
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": { "node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
@ -149,7 +149,7 @@
] ]
}, },
"package.json": { "package.json": {
"bytes": 2915, "bytes": 2989,
"imports": [] "imports": []
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
@ -235,6 +235,10 @@
"bytes": 19592, "bytes": 19592,
"imports": [] "imports": []
}, },
"src/gesture.js": {
"bytes": 2122,
"imports": []
},
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytes": 224151, "bytes": 224151,
"imports": [] "imports": []
@ -259,7 +263,7 @@
] ]
}, },
"src/handpose/handpipeline.js": { "src/handpose/handpipeline.js": {
"bytes": 7861, "bytes": 7959,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -294,7 +298,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15209, "bytes": 11168,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -315,7 +319,10 @@
"path": "src/handpose/handpose.js" "path": "src/handpose/handpose.js"
}, },
{ {
"path": "src/imagefx.js" "path": "src/gesture.js"
},
{
"path": "src/image.js"
}, },
{ {
"path": "src/profile.js" "path": "src/profile.js"
@ -328,6 +335,17 @@
} }
] ]
}, },
"src/image.js": {
"bytes": 4603,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/imagefx.js"
}
]
},
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19352, "bytes": 19352,
"imports": [] "imports": []
@ -481,7 +499,7 @@
"dist/human.esm.js.map": { "dist/human.esm.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5402750 "bytes": 5406771
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -627,23 +645,29 @@
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1090 "bytesInOutput": 1090
}, },
"src/gesture.js": {
"bytesInOutput": 1201
},
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11014 "bytesInOutput": 11014
}, },
"src/image.js": {
"bytesInOutput": 2395
},
"config.js": { "config.js": {
"bytesInOutput": 1273 "bytesInOutput": 1294
}, },
"package.json": { "package.json": {
"bytesInOutput": 2568 "bytesInOutput": 2642
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8484 "bytesInOutput": 5583
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 1273622 "bytes": 1274414
} }
} }
} }

492
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

44
dist/human.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"config.js": { "config.js": {
"bytes": 7216, "bytes": 7300,
"imports": [] "imports": []
}, },
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": { "node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
@ -149,7 +149,7 @@
] ]
}, },
"package.json": { "package.json": {
"bytes": 2915, "bytes": 2989,
"imports": [] "imports": []
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
@ -235,6 +235,10 @@
"bytes": 19592, "bytes": 19592,
"imports": [] "imports": []
}, },
"src/gesture.js": {
"bytes": 2122,
"imports": []
},
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytes": 224151, "bytes": 224151,
"imports": [] "imports": []
@ -259,7 +263,7 @@
] ]
}, },
"src/handpose/handpipeline.js": { "src/handpose/handpipeline.js": {
"bytes": 7861, "bytes": 7959,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -294,7 +298,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15209, "bytes": 11168,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -315,7 +319,10 @@
"path": "src/handpose/handpose.js" "path": "src/handpose/handpose.js"
}, },
{ {
"path": "src/imagefx.js" "path": "src/gesture.js"
},
{
"path": "src/image.js"
}, },
{ {
"path": "src/profile.js" "path": "src/profile.js"
@ -328,6 +335,17 @@
} }
] ]
}, },
"src/image.js": {
"bytes": 4603,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/imagefx.js"
}
]
},
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19352, "bytes": 19352,
"imports": [] "imports": []
@ -481,7 +499,7 @@
"dist/human.js.map": { "dist/human.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5402746 "bytes": 5406767
}, },
"dist/human.js": { "dist/human.js": {
"imports": [], "imports": [],
@ -627,20 +645,26 @@
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1090 "bytesInOutput": 1090
}, },
"src/gesture.js": {
"bytesInOutput": 1201
},
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11014 "bytesInOutput": 11014
}, },
"src/image.js": {
"bytesInOutput": 2395
},
"config.js": { "config.js": {
"bytesInOutput": 1273 "bytesInOutput": 1294
}, },
"package.json": { "package.json": {
"bytesInOutput": 2567 "bytesInOutput": 2641
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8522 "bytesInOutput": 5621
} }
}, },
"bytes": 1273667 "bytes": 1274459
} }
} }
} }

14
dist/human.node-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

492
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

45
dist/human.node.json vendored
View File

@ -1,11 +1,11 @@
{ {
"inputs": { "inputs": {
"config.js": { "config.js": {
"bytes": 7216, "bytes": 7300,
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2915, "bytes": 2989,
"imports": [] "imports": []
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
@ -74,6 +74,10 @@
"bytes": 19592, "bytes": 19592,
"imports": [] "imports": []
}, },
"src/gesture.js": {
"bytes": 2122,
"imports": []
},
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytes": 224151, "bytes": 224151,
"imports": [] "imports": []
@ -91,7 +95,7 @@
] ]
}, },
"src/handpose/handpipeline.js": { "src/handpose/handpipeline.js": {
"bytes": 7861, "bytes": 7959,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
@ -120,7 +124,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15209, "bytes": 11168,
"imports": [ "imports": [
{ {
"path": "src/facemesh/facemesh.js" "path": "src/facemesh/facemesh.js"
@ -138,7 +142,10 @@
"path": "src/handpose/handpose.js" "path": "src/handpose/handpose.js"
}, },
{ {
"path": "src/imagefx.js" "path": "src/gesture.js"
},
{
"path": "src/image.js"
}, },
{ {
"path": "src/profile.js" "path": "src/profile.js"
@ -151,6 +158,14 @@
} }
] ]
}, },
"src/image.js": {
"bytes": 4603,
"imports": [
{
"path": "src/imagefx.js"
}
]
},
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19352, "bytes": 19352,
"imports": [] "imports": []
@ -275,7 +290,7 @@
"dist/human.node-nobundle.js.map": { "dist/human.node-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 622579 "bytes": 622505
}, },
"dist/human.node-nobundle.js": { "dist/human.node-nobundle.js": {
"imports": [], "imports": [],
@ -302,7 +317,7 @@
"bytesInOutput": 9989 "bytesInOutput": 9989
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytesInOutput": 1254 "bytesInOutput": 1259
}, },
"src/profile.js": { "src/profile.js": {
"bytesInOutput": 619 "bytesInOutput": 619
@ -311,7 +326,7 @@
"bytesInOutput": 1236 "bytesInOutput": 1236
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1098 "bytesInOutput": 1104
}, },
"src/posenet/modelBase.js": { "src/posenet/modelBase.js": {
"bytesInOutput": 455 "bytesInOutput": 455
@ -364,23 +379,29 @@
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1112 "bytesInOutput": 1112
}, },
"src/gesture.js": {
"bytesInOutput": 1200
},
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11013 "bytesInOutput": 11013
}, },
"src/image.js": {
"bytesInOutput": 2379
},
"config.js": { "config.js": {
"bytesInOutput": 1271 "bytesInOutput": 1292
}, },
"package.json": { "package.json": {
"bytesInOutput": 2567 "bytesInOutput": 2641
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 28 "bytesInOutput": 28
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8467 "bytesInOutput": 5570
} }
}, },
"bytes": 212085 "bytes": 212875
} }
} }
} }

2
package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "0.6.6", "version": "0.6.6",
"description": "human: 3D Face Detection, Iris Tracking and Age & Gender Prediction", "description": "human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",
"module": "dist/human.esm.js", "module": "dist/human.esm.js",

45
src/gesture.js Normal file
View File

@ -0,0 +1,45 @@
exports.body = (res) => {
if (!res) return [];
const gestures = [];
for (const pose of res) {
// raising hands
const leftWrist = pose.keypoints.find((a) => (a.part === 'leftWrist'));
const rightWrist = pose.keypoints.find((a) => (a.part === 'rightWrist'));
const nose = pose.keypoints.find((a) => (a.part === 'nose'));
if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push('i give up');
else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push('raise left hand');
else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push('raise right hand');
// leaning
const leftShoulder = pose.keypoints.find((a) => (a.part === 'leftShoulder'));
const rightShoulder = pose.keypoints.find((a) => (a.part === 'rightShoulder'));
if (leftShoulder && rightShoulder) gestures.push(`leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}`);
}
return gestures;
};
exports.face = (res) => {
if (!res) return [];
const gestures = [];
for (const face of res) {
if (face.annotations['rightCheek'] && face.annotations['leftCheek'] && (face.annotations['rightCheek'].length > 0) && (face.annotations['leftCheek'].length > 0)) {
gestures.push(`facing ${((face.annotations['rightCheek'][0][2] > 0) || (face.annotations['leftCheek'][0][2] < 0)) ? 'right' : 'left'}`);
}
}
return gestures;
};
exports.hand = (res) => {
if (!res) return [];
const gestures = [];
for (const hand of res) {
const fingers = [];
for (const [finger, pos] of Object.entries(hand['annotations'])) {
if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger
}
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push(`${closest.name} forward ${highest.name} up`);
}
return gestures;
};

4
src/handpose/handpipeline.js
View File

@ -22,8 +22,8 @@ const util = require('./util');
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8; const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8;
const PALM_BOX_SHIFT_VECTOR = [0, -0.4]; const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
const PALM_BOX_ENLARGE_FACTOR = 3; const PALM_BOX_ENLARGE_FACTOR = 3;
const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
const HAND_BOX_ENLARGE_FACTOR = 1.65; const HAND_BOX_ENLARGE_FACTOR = 1.65; // increased from model default 1.65;
const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2]; const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2];
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0; const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2; const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;

115
src/human.js
View File

@ -4,7 +4,8 @@ const ssrnet = require('./ssrnet/ssrnet.js');
const emotion = require('./emotion/emotion.js'); const emotion = require('./emotion/emotion.js');
const posenet = require('./posenet/posenet.js'); const posenet = require('./posenet/posenet.js');
const handpose = require('./handpose/handpose.js'); const handpose = require('./handpose/handpose.js');
const fxImage = require('./imagefx.js'); const gesture = require('./gesture.js');
const image = require('./image.js');
const profile = require('./profile.js'); const profile = require('./profile.js');
const defaults = require('../config.js').default; const defaults = require('../config.js').default;
const app = require('../package.json'); const app = require('../package.json');
@ -52,9 +53,6 @@ class Human {
this.analyzeMemoryLeaks = false; this.analyzeMemoryLeaks = false;
this.checkSanity = false; this.checkSanity = false;
this.firstRun = true; this.firstRun = true;
// internal temp canvases
this.inCanvas = null;
this.outCanvas = null;
// object that contains all initialized models // object that contains all initialized models
this.models = { this.models = {
facemesh: null, facemesh: null,
@ -94,6 +92,7 @@ class Human {
if (leaked !== 0) this.log(...msg, leaked); if (leaked !== 0) this.log(...msg, leaked);
} }
// quick sanity check on inputs
sanity(input) { sanity(input) {
if (!this.checkSanity) return null; if (!this.checkSanity) return null;
if (!input) return 'input is not defined'; if (!input) return 'input is not defined';
@ -108,10 +107,12 @@ class Human {
return null; return null;
} }
// preload models, not explicitly required as it's done automatically on first use
async load(userConfig) { async load(userConfig) {
if (userConfig) this.config = mergeDeep(defaults, userConfig); if (userConfig) this.config = mergeDeep(defaults, userConfig);
if (this.firstRun) { if (this.firstRun) {
this.checkBackend(true);
this.log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`); this.log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`);
this.log('configuration:', this.config); this.log('configuration:', this.config);
this.log('flags:', tf.ENV.flags); this.log('flags:', tf.ENV.flags);
@ -144,8 +145,9 @@ class Human {
} }
} }
async checkBackend() { // check if backend needs initialization if it changed
if (tf.getBackend() !== this.config.backend) { async checkBackend(force) {
if (force || (tf.getBackend() !== this.config.backend)) {
this.state = 'backend'; this.state = 'backend';
/* force backend reload /* force backend reload
if (this.config.backend in tf.engine().registry) { if (this.config.backend in tf.engine().registry) {
@ -156,7 +158,7 @@ class Human {
this.log('Backend not registred:', this.config.backend); this.log('Backend not registred:', this.config.backend);
} }
*/ */
this.log('Setting backend:', this.config.backend); this.log('setting backend:', this.config.backend);
await tf.setBackend(this.config.backend); await tf.setBackend(this.config.backend);
tf.enableProdMode(); tf.enableProdMode();
/* debug mode is really too mcuh /* debug mode is really too mcuh
@ -167,84 +169,20 @@ class Human {
this.log('Changing WebGL: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate); this.log('Changing WebGL: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1); tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
} }
tf.ENV.set('WEBGL_CPU_FORWARD', true); // tf.ENV.set('WEBGL_CPU_FORWARD', true);
// tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
// tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
await tf.ready(); await tf.ready();
} }
} }
tfImage(input) { // main detect function
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
const originalHeight = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0));
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0) targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0) targetWidth = originalWidth * (this.config.filter.height / originalHeight);
if (this.config.filter.height > 0) targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0) targetHeight = originalHeight * (this.config.filter.width / originalWidth);
if (!this.inCanvas || (this.inCanvas.width !== targetWidth) || (this.inCanvas.height !== targetHeight)) {
this.inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
if (this.inCanvas.width !== targetWidth) this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight) this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext('2d');
if (input instanceof ImageData) ctx.putImageData(input, 0, 0);
else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.fx || !this.outCanvas || (this.inCanvas.width !== this.outCanvas.width) || (this.inCanvas.height !== this.outCanvas.height)) {
this.outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement('canvas');
if (this.outCanvas.width !== this.inCanvas.width) this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height) this.outCanvas.height = this.inCanvas.height;
this.fx = (tf.ENV.flags.IS_BROWSER && (typeof document !== 'undefined')) ? new fxImage.Canvas({ canvas: this.outCanvas }) : null;
}
this.fx.reset();
this.fx.addFilter('brightness', this.config.filter.brightness); // must have at least one filter enabled
if (this.config.filter.contrast !== 0) this.fx.addFilter('contrast', this.config.filter.contrast);
if (this.config.filter.sharpness !== 0) this.fx.addFilter('sharpen', this.config.filter.sharpness);
if (this.config.filter.blur !== 0) this.fx.addFilter('blur', this.config.filter.blur);
if (this.config.filter.saturation !== 0) this.fx.addFilter('saturation', this.config.filter.saturation);
if (this.config.filter.hue !== 0) this.fx.addFilter('hue', this.config.filter.hue);
if (this.config.filter.negative) this.fx.addFilter('negative');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.vintage) this.fx.addFilter('brownie');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.kodachrome) this.fx.addFilter('kodachrome');
if (this.config.filter.technicolor) this.fx.addFilter('technicolor');
if (this.config.filter.polaroid) this.fx.addFilter('polaroid');
if (this.config.filter.pixelate !== 0) this.fx.addFilter('pixelate', this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas) this.outCanvas = this.inCanvas;
let pixels;
if ((this.config.backend === 'webgl') || (this.outCanvas instanceof ImageData)) {
// tf kernel-optimized method to get imagedata, also if input is imagedata, just use it
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
// cpu and wasm kernel does not implement efficient fromPixels method nor we can use canvas as-is, so we do a silly one more canvas
const tempCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext('2d');
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
const casted = pixels.toFloat();
tensor = casted.expandDims(0);
pixels.dispose();
casted.dispose();
}
return { tensor, canvas: this.config.filter.return ? this.outCanvas : null };
}
async detect(input, userConfig = {}) { async detect(input, userConfig = {}) {
this.state = 'config'; this.state = 'config';
const perf = {}; const perf = {};
let timeStamp; let timeStamp;
// update configuration
this.config = mergeDeep(defaults, userConfig); this.config = mergeDeep(defaults, userConfig);
if (!this.config.videoOptimized) this.config = mergeDeep(this.config, override); if (!this.config.videoOptimized) this.config = mergeDeep(this.config, override);
@ -256,6 +194,7 @@ class Human {
return { error }; return { error };
} }
// detection happens inside a promise
// eslint-disable-next-line no-async-promise-executor // eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
let poseRes; let poseRes;
@ -281,9 +220,8 @@ class Human {
this.analyze('Start Detect:'); this.analyze('Start Detect:');
timeStamp = now(); timeStamp = now();
const image = this.tfImage(input); const process = image.process(input, this.config);
perf.image = Math.trunc(now() - timeStamp); perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
// run facemesh, includes blazeface and iris // run facemesh, includes blazeface and iris
const faceRes = []; const faceRes = [];
@ -291,7 +229,7 @@ class Human {
this.state = 'run:face'; this.state = 'run:face';
timeStamp = now(); timeStamp = now();
this.analyze('Start FaceMesh:'); this.analyze('Start FaceMesh:');
const faces = await this.models.facemesh.estimateFaces(imageTensor, this.config.face); const faces = await this.models.facemesh.estimateFaces(process.tensor, this.config.face);
perf.face = Math.trunc(now() - timeStamp); perf.face = Math.trunc(now() - timeStamp);
for (const face of faces) { for (const face of faces) {
// is something went wrong, skip the face // is something went wrong, skip the face
@ -334,38 +272,45 @@ class Human {
// run posenet // run posenet
if (this.config.async) { if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : []; poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(process.tensor, this.config.body) : [];
} else { } else {
this.state = 'run:body'; this.state = 'run:body';
timeStamp = now(); timeStamp = now();
this.analyze('Start PoseNet'); this.analyze('Start PoseNet');
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : []; poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(process.tensor, this.config.body) : [];
this.analyze('End PoseNet:'); this.analyze('End PoseNet:');
perf.body = Math.trunc(now() - timeStamp); perf.body = Math.trunc(now() - timeStamp);
} }
// run handpose // run handpose
if (this.config.async) { if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : []; handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(process.tensor, this.config.hand) : [];
} else { } else {
this.state = 'run:hand'; this.state = 'run:hand';
timeStamp = now(); timeStamp = now();
this.analyze('Start HandPose:'); this.analyze('Start HandPose:');
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : []; handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(process.tensor, this.config.hand) : [];
this.analyze('End HandPose:'); this.analyze('End HandPose:');
perf.hand = Math.trunc(now() - timeStamp); perf.hand = Math.trunc(now() - timeStamp);
} }
if (this.config.async) [poseRes, handRes] = await Promise.all([poseRes, handRes]); if (this.config.async) [poseRes, handRes] = await Promise.all([poseRes, handRes]);
imageTensor.dispose(); process.tensor.dispose();
this.state = 'idle'; this.state = 'idle';
if (this.config.scoped) tf.engine().endScope(); if (this.config.scoped) tf.engine().endScope();
this.analyze('End Scope:'); this.analyze('End Scope:');
let gestureRes = [];
if (this.config.gesture.enabled) {
timeStamp = now();
gestureRes = { body: gesture.body(poseRes), hand: gesture.hand(handRes), face: gesture.face(faceRes) };
perf.gesture = Math.trunc(now() - timeStamp);
}
perf.total = Math.trunc(now() - timeStart); perf.total = Math.trunc(now() - timeStart);
resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf, canvas: image.canvas }); resolve({ face: faceRes, body: poseRes, hand: handRes, gesture: gestureRes, performance: perf, canvas: process.canvas });
}); });
} }
} }

79
src/image.js Normal file
View File

@ -0,0 +1,79 @@
const tf = require('@tensorflow/tfjs');
const fxImage = require('./imagefx.js');
// internal temp canvases
let inCanvas = null;
let outCanvas = null;
// process input image and return tensor
// input can be tensor, imagedata, htmlimageelement, htmlvideoelement
// input is resized and run through imagefx filter
function process(input, config) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
const originalHeight = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0));
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (config.filter.width > 0) targetWidth = config.filter.width;
else if (config.filter.height > 0) targetWidth = originalWidth * (config.filter.height / originalHeight);
if (config.filter.height > 0) targetHeight = config.filter.height;
else if (config.filter.width > 0) targetHeight = originalHeight * (config.filter.width / originalWidth);
if (!inCanvas || (inCanvas.width !== targetWidth) || (inCanvas.height !== targetHeight)) {
inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
if (inCanvas.width !== targetWidth) inCanvas.width = targetWidth;
if (inCanvas.height !== targetHeight) inCanvas.height = targetHeight;
}
const ctx = inCanvas.getContext('2d');
if (input instanceof ImageData) ctx.putImageData(input, 0, 0);
else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, inCanvas.width, inCanvas.height);
if (config.filter.enabled) {
if (!this.fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas.height !== outCanvas.height)) {
outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(inCanvas.width, inCanvas.height) : document.createElement('canvas');
if (outCanvas.width !== inCanvas.width) outCanvas.width = inCanvas.width;
if (outCanvas.height !== inCanvas.height) outCanvas.height = inCanvas.height;
this.fx = (tf.ENV.flags.IS_BROWSER && (typeof document !== 'undefined')) ? new fxImage.Canvas({ canvas: outCanvas }) : null;
}
this.fx.reset();
this.fx.addFilter('brightness', config.filter.brightness); // must have at least one filter enabled
if (config.filter.contrast !== 0) this.fx.addFilter('contrast', config.filter.contrast);
if (config.filter.sharpness !== 0) this.fx.addFilter('sharpen', config.filter.sharpness);
if (config.filter.blur !== 0) this.fx.addFilter('blur', config.filter.blur);
if (config.filter.saturation !== 0) this.fx.addFilter('saturation', config.filter.saturation);
if (config.filter.hue !== 0) this.fx.addFilter('hue', config.filter.hue);
if (config.filter.negative) this.fx.addFilter('negative');
if (config.filter.sepia) this.fx.addFilter('sepia');
if (config.filter.vintage) this.fx.addFilter('brownie');
if (config.filter.sepia) this.fx.addFilter('sepia');
if (config.filter.kodachrome) this.fx.addFilter('kodachrome');
if (config.filter.technicolor) this.fx.addFilter('technicolor');
if (config.filter.polaroid) this.fx.addFilter('polaroid');
if (config.filter.pixelate !== 0) this.fx.addFilter('pixelate', config.filter.pixelate);
this.fx.apply(inCanvas);
}
if (!outCanvas) outCanvas = inCanvas;
let pixels;
if ((config.backend === 'webgl') || (outCanvas instanceof ImageData)) {
// tf kernel-optimized method to get imagedata, also if input is imagedata, just use it
pixels = tf.browser.fromPixels(outCanvas);
} else {
// cpu and wasm kernel does not implement efficient fromPixels method nor we can use canvas as-is, so we do a silly one more canvas
const tempCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext('2d');
tempCtx.drawImage(outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
const casted = pixels.toFloat();
tensor = casted.expandDims(0);
pixels.dispose();
casted.dispose();
}
return { tensor, canvas: config.filter.return ? outCanvas : null };
}
exports.process = process;