ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update hand algorithm

pull/50/head
Vladimir Mandic 2020-11-08 01:17:25 -05:00
parent 8b7594b459
commit 1c0499d797
21 changed files with 143 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.js
View File

@ -109,10 +109,10 @@ export default {
// if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis // if model is running st 25 FPS, we can re-use existing bounding box for updated hand skeleton analysis
// as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec) // as the hand probably hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.5, // threshold for discarding a prediction minConfidence: 0.5, // threshold for discarding a prediction
iouThreshold: 0.2, // threshold for deciding whether boxes overlap too much in non-maximum suppression iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in non-maximum suppression
scoreThreshold: 0.5, // threshold for deciding when to remove boxes based on score in non-maximum suppression scoreThreshold: 0.8, // threshold for deciding when to remove boxes based on score in non-maximum suppression
enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace enlargeFactor: 1.65, // empiric tuning as skeleton prediction prefers hand box with some whitespace
maxHands: 10, // maximum number of hands detected in the input, should be set to the minimum number for performance maxHands: 1, // maximum number of hands detected in the input, should be set to the minimum number for performance
detector: { detector: {
modelPath: '../models/handdetect.json', modelPath: '../models/handdetect.json',
}, },

2
dev-server.js
View File

@ -61,7 +61,7 @@ async function build(f, msg) {
if (!es) es = await esbuild.startService(); if (!es) es = await esbuild.startService();
// common build options // common build options
const cfg = { const cfg = {
minify: true, minify: false,
bundle: true, bundle: true,
sourcemap: true, sourcemap: true,
logLevel: 'error', logLevel: 'error',

70
dist/demo-browser-index.js vendored
View File

@ -32769,25 +32769,22 @@ var Mx = we((Px) => {
return et.mul(e, this.inputSizeTensor); return et.mul(e, this.inputSizeTensor);
}); });
} }
async getBoundingBoxes(n, t) { async getBoxes(n, t) {
const e = this.model.predict(n), r = e.squeeze(), i = et.tidy(() => et.sigmoid(et.slice(r, [0, 0], [-1, 1])).squeeze()), a = et.slice(r, [0, 1], [-1, 4]), s = this.normalizeBoxes(a), o = await et.image.nonMaxSuppressionAsync(s, i, t.maxHands, t.iouThreshold, t.scoreThreshold), c = o.arraySync(), l = [e, o, r, s, a, i]; const e = this.model.predict(n), r = e.squeeze(), i = et.tidy(() => et.sigmoid(et.slice(r, [0, 0], [-1, 1])).squeeze()), a = et.slice(r, [0, 1], [-1, 4]), s = this.normalizeBoxes(a), o = await et.image.nonMaxSuppressionAsync(s, i, t.maxHands, t.iouThreshold, t.scoreThreshold), c = o.arraySync(), l = [e, o, r, s, a, i], u = [];
if (c.length === 0)
return l.forEach((h) => h.dispose()), null;
const u = [];
for (const h of c) { for (const h of c) {
const d = et.slice(s, [h, 0], [1, -1]), p = et.slice(r, [h, 5], [1, 14]), f = et.tidy(() => this.normalizeLandmarks(p, h).reshape([-1, 2])); const d = et.slice(s, [h, 0], [1, -1]), p = et.slice(r, [h, 5], [1, 14]), f = et.tidy(() => this.normalizeLandmarks(p, h).reshape([-1, 2]));
p.dispose(), u.push({boxes: d, palmLandmarks: f}); p.dispose(), u.push({box: d, palmLandmarks: f});
} }
return l.forEach((h) => h.dispose()), u; return l.forEach((h) => h.dispose()), u;
} }
async estimateHandBounds(n, t) { async estimateHandBounds(n, t) {
const e = n.shape[1], r = n.shape[2], i = et.tidy(() => n.resizeBilinear([t.inputSize, t.inputSize]).div(127.5).sub(1)), a = await this.getBoundingBoxes(i, t); const e = n.shape[1], r = n.shape[2], i = et.tidy(() => n.resizeBilinear([t.inputSize, t.inputSize]).div(127.5).sub(1)), a = await this.getBoxes(i, t);
if (i.dispose(), !a || a.length === 0) if (i.dispose(), !a || a.length === 0)
return null; return null;
const s = []; const s = [];
for (const o of a) { for (const o of a) {
const c = o.boxes.dataSync(), l = c.slice(0, 2), u = c.slice(2, 4), h = o.palmLandmarks.arraySync(); const c = o.box.dataSync(), l = c.slice(0, 2), u = c.slice(2, 4), h = o.palmLandmarks.arraySync();
o.boxes.dispose(), o.palmLandmarks.dispose(), s.push(UV.scaleBoxCoordinates({startPoint: l, endPoint: u, palmLandmarks: h}, [r / t.inputSize, e / t.inputSize])); o.box.dispose(), o.palmLandmarks.dispose(), s.push(UV.scaleBoxCoordinates({startPoint: l, endPoint: u, palmLandmarks: h}, [r / t.inputSize, e / t.inputSize]));
} }
return s; return s;
} }
@ -32841,7 +32838,7 @@ var Xx = we((Kx) => {
const jx = Ut(), Vn = Jp(), Wr = Yx(), GV = 0.8, qV = [0, -0.4], YV = 3, KV = [0, -0.1], jV = 1.65, $x = [0, 5, 9, 13, 17, 1, 2], $V = 0, XV = 2; const jx = Ut(), Vn = Jp(), Wr = Yx(), GV = 0.8, qV = [0, -0.4], YV = 3, KV = [0, -0.1], jV = 1.65, $x = [0, 5, 9, 13, 17, 1, 2], $V = 0, XV = 2;
class JV { class JV {
constructor(n, t, e) { constructor(n, t, e) {
this.boundingBoxDetector = n, this.meshDetector = t, this.inputSize = e, this.regionsOfInterest = [], this.runsWithoutHandDetector = 0, this.skipFrames = 0, this.detectedHands = 0; this.boxDetector = n, this.meshDetector = t, this.inputSize = e, this.storedBoxes = [], this.skipped = 0, this.detectedHands = 0;
} }
getBoxForPalmLandmarks(n, t) { getBoxForPalmLandmarks(n, t) {
const e = n.map((i) => { const e = n.map((i) => {
@ -32864,51 +32861,50 @@ var Xx = we((Kx) => {
return c.map((d) => [d[0] + h[0], d[1] + h[1], d[2]]); return c.map((d) => [d[0] + h[0], d[1] + h[1], d[2]]);
} }
async estimateHands(n, t) { async estimateHands(n, t) {
this.skipFrames = t.skipFrames; this.skipped++;
let e = this.runsWithoutHandDetector > this.skipFrames || this.detectedHands !== this.regionsOfInterest.length, r; let e = false;
if (e && (r = await this.boundingBoxDetector.estimateHandBounds(n, t)), t.maxHands > 1 && r && r.length > 0 && r.length !== this.detectedHands && (e = true), e) { const r = this.skipped > t.skipFrames ? await this.boxDetector.estimateHandBounds(n, t) : null;
if (this.regionsOfInterest = [], !r || r.length === 0) if (r && r.length !== this.detectedHands && this.detectedHands !== t.maxHands) {
return this.detectedHands = 0, null; this.storedBoxes = [], this.detectedHands = 0;
for (const a of r) for (const a of r)
this.regionsOfInterest.push(a); this.storedBoxes.push(a);
this.runsWithoutHandDetector = 0; this.storedBoxes.length > 0 && (e = true), this.skipped = 0;
} else }
this.runsWithoutHandDetector++;
const i = []; const i = [];
for (const a in this.regionsOfInterest) { for (const a in this.storedBoxes) {
const s = this.regionsOfInterest[a]; const s = this.storedBoxes[a];
if (!s) if (!s)
continue; continue;
const o = Wr.computeRotation(s.palmLandmarks[$V], s.palmLandmarks[XV]), c = Vn.getBoxCenter(s), l = [c[0] / n.shape[2], c[1] / n.shape[1]], u = jx.image.rotateWithOffset(n, o, 0, l), h = Wr.buildRotationMatrix(-o, c), d = e ? this.getBoxForPalmLandmarks(s.palmLandmarks, h) : s, p = Vn.cutBoxFromImageAndResize(d, u, [this.inputSize, this.inputSize]), f = p.div(255); const o = Wr.computeRotation(s.palmLandmarks[$V], s.palmLandmarks[XV]), c = Vn.getBoxCenter(s), l = [c[0] / n.shape[2], c[1] / n.shape[1]], u = jx.image.rotateWithOffset(n, o, 0, l), h = Wr.buildRotationMatrix(-o, c), d = e ? this.getBoxForPalmLandmarks(s.palmLandmarks, h) : s, p = Vn.cutBoxFromImageAndResize(d, u, [this.inputSize, this.inputSize]), f = p.div(255);
p.dispose(), u.dispose(); p.dispose(), u.dispose();
const m = this.meshDetector.predict(f), [g, y] = m; const [m, g] = await this.meshDetector.predict(f);
f.dispose(); f.dispose();
const w = g.dataSync()[0]; const y = m.dataSync()[0];
if (g.dispose(), w >= t.minConfidence) { if (m.dispose(), y >= t.minConfidence) {
const b = jx.reshape(y, [-1, 3]), L = b.arraySync(); const w = jx.reshape(g, [-1, 3]), b = w.arraySync();
y.dispose(), b.dispose(); g.dispose(), w.dispose();
const x = this.transformRawCoords(L, d, o, h), N = this.getBoxForHandLandmarks(x); const L = this.transformRawCoords(b, d, o, h), x = this.getBoxForHandLandmarks(L);
this.updateRegionsOfInterest(N, a); this.updateStoredBoxes(x, a);
const I = {landmarks: x, handInViewConfidence: w, boundingBox: {topLeft: N.startPoint, bottomRight: N.endPoint}}; const N = {landmarks: L, handInViewConfidence: y, boundingBox: {topLeft: x.startPoint, bottomRight: x.endPoint}};
i.push(I); i.push(N);
} else } else
this.updateRegionsOfInterest(null, a); this.updateStoredBoxes(null, a);
y.dispose(); g.dispose();
} }
return this.regionsOfInterest = this.regionsOfInterest.filter((a) => a !== null), this.detectedHands = i.length, i; return this.storedBoxes = this.storedBoxes.filter((a) => a !== null), this.detectedHands = i.length, i;
} }
calculateLandmarksBoundingBox(n) { calculateLandmarksBoundingBox(n) {
const t = n.map((a) => a[0]), e = n.map((a) => a[1]), r = [Math.min(...t), Math.min(...e)], i = [Math.max(...t), Math.max(...e)]; const t = n.map((a) => a[0]), e = n.map((a) => a[1]), r = [Math.min(...t), Math.min(...e)], i = [Math.max(...t), Math.max(...e)];
return {startPoint: r, endPoint: i}; return {startPoint: r, endPoint: i};
} }
updateRegionsOfInterest(n, t) { updateStoredBoxes(n, t) {
const e = this.regionsOfInterest[t]; const e = this.storedBoxes[t];
let r = 0; let r = 0;
if (n && e && e.startPoint) { if (n && e && e.startPoint) {
const [i, a] = n.startPoint, [s, o] = n.endPoint, [c, l] = e.startPoint, [u, h] = e.endPoint, d = Math.max(i, c), p = Math.max(a, l), f = Math.min(s, u), m = Math.min(o, h), g = (f - d) * (m - p), y = (s - i) * (o - a), w = (u - c) * (h - a); const [i, a] = n.startPoint, [s, o] = n.endPoint, [c, l] = e.startPoint, [u, h] = e.endPoint, d = Math.max(i, c), p = Math.max(a, l), f = Math.min(s, u), m = Math.min(o, h), g = (f - d) * (m - p), y = (s - i) * (o - a), w = (u - c) * (h - a);
r = g / (y + w - g); r = g / (y + w - g);
} }
this.regionsOfInterest[t] = r > GV ? e : n; this.storedBoxes[t] = r > GV ? e : n;
} }
} }
Kx.HandPipeline = JV; Kx.HandPipeline = JV;
@ -33159,7 +33155,7 @@ var sL = we((aL) => {
}); });
var oL = we((sG) => { var oL = we((sG) => {
Is(sG, {default: () => oG}); Is(sG, {default: () => oG});
var oG = {backend: "webgl", console: true, async: true, profile: false, deallocate: false, scoped: false, videoOptimized: true, filter: {enabled: true, width: 0, height: 0, return: true, brightness: 0, contrast: 0, sharpness: 0, blur: 0, saturation: 0, hue: 0, negative: false, sepia: false, vintage: false, kodachrome: false, technicolor: false, polaroid: false, pixelate: 0}, gesture: {enabled: true}, face: {enabled: true, detector: {modelPath: "../models/blazeface-back.json", inputSize: 256, maxFaces: 10, skipFrames: 15, minConfidence: 0.1, iouThreshold: 0.1, scoreThreshold: 0.2}, mesh: {enabled: true, modelPath: "../models/facemesh.json", inputSize: 192}, iris: {enabled: true, modelPath: "../models/iris.json", enlargeFactor: 2.3, inputSize: 64}, age: {enabled: true, modelPath: "../models/age-ssrnet-imdb.json", inputSize: 64, skipFrames: 15}, gender: {enabled: true, minConfidence: 0.1, modelPath: "../models/gender-ssrnet-imdb.json", inputSize: 64, skipFrames: 15}, emotion: {enabled: true, inputSize: 64, minConfidence: 0.2, skipFrames: 15, modelPath: "../models/emotion-large.json"}}, body: {enabled: true, modelPath: "../models/posenet.json", inputResolution: 257, outputStride: 16, maxDetections: 10, scoreThreshold: 0.8, nmsRadius: 20}, hand: {enabled: true, inputSize: 256, skipFrames: 15, minConfidence: 0.5, iouThreshold: 0.2, scoreThreshold: 0.5, enlargeFactor: 1.65, maxHands: 10, detector: {modelPath: "../models/handdetect.json"}, skeleton: {modelPath: "../models/handskeleton.json"}}}; var oG = {backend: "webgl", console: true, async: true, profile: false, deallocate: false, scoped: false, videoOptimized: true, filter: {enabled: true, width: 0, height: 0, return: true, brightness: 0, contrast: 0, sharpness: 0, blur: 0, saturation: 0, hue: 0, negative: false, sepia: false, vintage: false, kodachrome: false, technicolor: false, polaroid: false, pixelate: 0}, gesture: {enabled: true}, face: {enabled: true, detector: {modelPath: "../models/blazeface-back.json", inputSize: 256, maxFaces: 10, skipFrames: 15, minConfidence: 0.1, iouThreshold: 0.1, scoreThreshold: 0.2}, mesh: {enabled: true, modelPath: "../models/facemesh.json", inputSize: 192}, iris: {enabled: true, modelPath: "../models/iris.json", enlargeFactor: 2.3, inputSize: 64}, age: {enabled: true, modelPath: "../models/age-ssrnet-imdb.json", inputSize: 64, skipFrames: 15}, gender: {enabled: true, minConfidence: 0.1, modelPath: "../models/gender-ssrnet-imdb.json", inputSize: 64, skipFrames: 15}, emotion: {enabled: true, inputSize: 64, minConfidence: 0.2, skipFrames: 15, modelPath: "../models/emotion-large.json"}}, body: {enabled: true, modelPath: "../models/posenet.json", inputResolution: 257, outputStride: 16, maxDetections: 10, scoreThreshold: 0.8, nmsRadius: 20}, hand: {enabled: true, inputSize: 256, skipFrames: 15, minConfidence: 0.5, iouThreshold: 0.1, scoreThreshold: 0.8, enlargeFactor: 1.65, maxHands: 1, detector: {modelPath: "../models/handdetect.json"}, skeleton: {modelPath: "../models/handskeleton.json"}}};
}); });
var lL = we((pq, cL) => { var lL = we((pq, cL) => {
cL.exports = {name: "@vladmandic/human", version: "0.8.1", description: "human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition", sideEffects: false, main: "dist/human.node.js", module: "dist/human.esm.js", browser: "dist/human.esm.js", author: "Vladimir Mandic <mandic00@live.com>", bugs: {url: "https://github.com/vladmandic/human/issues"}, homepage: "https://github.com/vladmandic/human#readme", license: "MIT", engines: {node: ">=14.0.0"}, repository: {type: "git", url: "git+https://github.com/vladmandic/human.git"}, dependencies: {}, peerDependencies: {}, devDependencies: {"@tensorflow/tfjs": "^2.7.0", "@tensorflow/tfjs-node": "^2.7.0", "@vladmandic/pilogger": "^0.2.7", chokidar: "^3.4.3", dayjs: "^1.9.5", esbuild: "^0.7.22", eslint: "^7.13.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", rimraf: "^3.0.2", seedrandom: "^3.0.5", "simple-git": "^2.21.0"}, scripts: {start: "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js", lint: "eslint src/*.js demo/*.js", dev: "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server.js", "build-iife": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=iife --external:fs --global-name=Human --metafile=dist/human.json --outfile=dist/human.js src/human.js", "build-esm-bundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js", "build-esm-nobundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js", "build-node": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --metafile=dist/human.node.json --outfile=dist/human.node.js src/human.js", "build-node-nobundle": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --external:@tensorflow --metafile=dist/human.node.json --outfile=dist/human.node-nobundle.js src/human.js", "build-demo": "esbuild --bundle --log-level=error --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/demo-browser-index.json --outfile=dist/demo-browser-index.js demo/browser.js", build: "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo", update: "npm update --depth 20 --force && npm dedupe && npm prune && npm audit", changelog: "node changelog.js"}, keywords: ["tensorflowjs", "face-detection", "face-geometry", "body-tracking", "hand-tracking", "iris-tracking", "age-estimation", "emotion-detection", "gender-prediction", "gesture-recognition"]}; cL.exports = {name: "@vladmandic/human", version: "0.8.1", description: "human: 3D Face Detection, Body Pose, Hand & Finger Tracking, Iris Tracking, Age & Gender Prediction, Emotion Prediction & Gesture Recognition", sideEffects: false, main: "dist/human.node.js", module: "dist/human.esm.js", browser: "dist/human.esm.js", author: "Vladimir Mandic <mandic00@live.com>", bugs: {url: "https://github.com/vladmandic/human/issues"}, homepage: "https://github.com/vladmandic/human#readme", license: "MIT", engines: {node: ">=14.0.0"}, repository: {type: "git", url: "git+https://github.com/vladmandic/human.git"}, dependencies: {}, peerDependencies: {}, devDependencies: {"@tensorflow/tfjs": "^2.7.0", "@tensorflow/tfjs-node": "^2.7.0", "@vladmandic/pilogger": "^0.2.7", chokidar: "^3.4.3", dayjs: "^1.9.5", esbuild: "^0.7.22", eslint: "^7.13.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", rimraf: "^3.0.2", seedrandom: "^3.0.5", "simple-git": "^2.21.0"}, scripts: {start: "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js", lint: "eslint src/*.js demo/*.js", dev: "npm install && node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation dev-server.js", "build-iife": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=iife --external:fs --global-name=Human --metafile=dist/human.json --outfile=dist/human.js src/human.js", "build-esm-bundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/human.esm.json --outfile=dist/human.esm.js src/human.js", "build-esm-nobundle": "esbuild --bundle --minify --platform=browser --sourcemap --target=es2018 --format=esm --external:@tensorflow --external:fs --metafile=dist/human.esm-nobundle.json --outfile=dist/human.esm-nobundle.js src/human.js", "build-node": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --metafile=dist/human.node.json --outfile=dist/human.node.js src/human.js", "build-node-nobundle": "esbuild --bundle --minify --platform=node --sourcemap --target=es2018 --format=cjs --external:@tensorflow --metafile=dist/human.node.json --outfile=dist/human.node-nobundle.js src/human.js", "build-demo": "esbuild --bundle --log-level=error --platform=browser --sourcemap --target=es2018 --format=esm --external:fs --metafile=dist/demo-browser-index.json --outfile=dist/demo-browser-index.js demo/browser.js", build: "rimraf dist/* && npm run build-iife && npm run build-esm-bundle && npm run build-esm-nobundle && npm run build-node && npm run build-node-nobundle && npm run build-demo", update: "npm update --depth 20 --force && npm dedupe && npm prune && npm audit", changelog: "node changelog.js"}, keywords: ["tensorflowjs", "face-detection", "face-geometry", "body-tracking", "hand-tracking", "iris-tracking", "age-estimation", "emotion-detection", "gender-prediction", "gesture-recognition"]};

4
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
dist/demo-browser-index.json vendored
View File

@ -23,7 +23,7 @@
"imports": [] "imports": []
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1278535, "bytes": 1278200,
"imports": [] "imports": []
} }
}, },
@ -31,13 +31,13 @@
"dist/demo-browser-index.js.map": { "dist/demo-browser-index.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5533079 "bytes": 5532275
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1665000 "bytesInOutput": 1664606
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 8716 "bytesInOutput": 8716
@ -52,7 +52,7 @@
"bytesInOutput": 15855 "bytesInOutput": 15855
} }
}, },
"bytes": 1709822 "bytes": 1709428
} }
} }
} }

14
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.esm-nobundle.json vendored
View File

@ -206,7 +206,7 @@
"imports": [] "imports": []
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytes": 4316, "bytes": 4220,
"imports": [ "imports": [
{ {
"path": "src/hand/box.js" "path": "src/hand/box.js"
@ -214,7 +214,7 @@
] ]
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytes": 8724, "bytes": 8214,
"imports": [ "imports": [
{ {
"path": "src/hand/box.js" "path": "src/hand/box.js"
@ -301,7 +301,7 @@
"dist/human.esm-nobundle.js.map": { "dist/human.esm-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 623550 "bytes": 622741
}, },
"dist/human.esm-nobundle.js": { "dist/human.esm-nobundle.js": {
"imports": [], "imports": [],
@ -379,13 +379,13 @@
"bytesInOutput": 1420 "bytesInOutput": 1420
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 1830 "bytesInOutput": 1748
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 997 "bytesInOutput": 997
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 3130 "bytesInOutput": 2878
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127000 "bytesInOutput": 127000
@ -403,7 +403,7 @@
"bytesInOutput": 2349 "bytesInOutput": 2349
}, },
"config.js": { "config.js": {
"bytesInOutput": 1326 "bytesInOutput": 1325
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3004
@ -415,7 +415,7 @@
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 216865 "bytes": 216530
} }
} }
} }

4
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.esm.json vendored
View File

@ -387,7 +387,7 @@
] ]
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytes": 4316, "bytes": 4220,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -398,7 +398,7 @@
] ]
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytes": 8724, "bytes": 8214,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,7 +513,7 @@
"dist/human.esm.js.map": { "dist/human.esm.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5419535 "bytes": 5418726
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -648,13 +648,13 @@
"bytesInOutput": 1398 "bytesInOutput": 1398
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 1836 "bytesInOutput": 1754
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 1005 "bytesInOutput": 1005
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 3128 "bytesInOutput": 2876
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127001 "bytesInOutput": 127001
@ -672,7 +672,7 @@
"bytesInOutput": 2365 "bytesInOutput": 2365
}, },
"config.js": { "config.js": {
"bytesInOutput": 1327 "bytesInOutput": 1326
}, },
"package.json": { "package.json": {
"bytesInOutput": 3005 "bytesInOutput": 3005
@ -684,7 +684,7 @@
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 1278535 "bytes": 1278200
} }
} }
} }

4
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.json vendored
View File

@ -387,7 +387,7 @@
] ]
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytes": 4316, "bytes": 4220,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -398,7 +398,7 @@
] ]
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytes": 8724, "bytes": 8214,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,7 +513,7 @@
"dist/human.js.map": { "dist/human.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5419531 "bytes": 5418722
}, },
"dist/human.js": { "dist/human.js": {
"imports": [], "imports": [],
@ -648,13 +648,13 @@
"bytesInOutput": 1398 "bytesInOutput": 1398
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 1836 "bytesInOutput": 1754
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 1005 "bytesInOutput": 1005
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 3128 "bytesInOutput": 2876
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127001 "bytesInOutput": 127001
@ -672,7 +672,7 @@
"bytesInOutput": 2365 "bytesInOutput": 2365
}, },
"config.js": { "config.js": {
"bytesInOutput": 1327 "bytesInOutput": 1326
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3004
@ -681,7 +681,7 @@
"bytesInOutput": 7257 "bytesInOutput": 7257
} }
}, },
"bytes": 1278580 "bytes": 1278245
} }
} }
} }

14
dist/human.node-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

14
dist/human.node.json vendored
View File

@ -206,7 +206,7 @@
"imports": [] "imports": []
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytes": 4316, "bytes": 4220,
"imports": [ "imports": [
{ {
"path": "src/hand/box.js" "path": "src/hand/box.js"
@ -214,7 +214,7 @@
] ]
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytes": 8724, "bytes": 8214,
"imports": [ "imports": [
{ {
"path": "src/hand/box.js" "path": "src/hand/box.js"
@ -301,7 +301,7 @@
"dist/human.node-nobundle.js.map": { "dist/human.node-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 637683 "bytes": 636874
}, },
"dist/human.node-nobundle.js": { "dist/human.node-nobundle.js": {
"imports": [], "imports": [],
@ -379,13 +379,13 @@
"bytesInOutput": 1419 "bytesInOutput": 1419
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 1830 "bytesInOutput": 1748
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 996 "bytesInOutput": 996
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 3130 "bytesInOutput": 2878
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127000 "bytesInOutput": 127000
@ -403,7 +403,7 @@
"bytesInOutput": 2349 "bytesInOutput": 2349
}, },
"config.js": { "config.js": {
"bytesInOutput": 1325 "bytesInOutput": 1324
}, },
"package.json": { "package.json": {
"bytesInOutput": 3004 "bytesInOutput": 3004
@ -415,7 +415,7 @@
"bytesInOutput": 7201 "bytesInOutput": 7201
} }
}, },
"bytes": 216872 "bytes": 216537
} }
} }
} }

35
src/hand/handdetector.js
View File

@ -46,33 +46,30 @@ class HandDetector {
}); });
} }
async getBoundingBoxes(input, config) { async getBoxes(input, config) {
const batchedPrediction = this.model.predict(input); const batched = this.model.predict(input);
const prediction = batchedPrediction.squeeze(); const predictions = batched.squeeze();
const scores = tf.tidy(() => tf.sigmoid(tf.slice(prediction, [0, 0], [-1, 1])).squeeze()); const scores = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze());
const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]); // const scoresVal = scores.dataSync(); // scoresVal[boxIndex] is box confidence
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold); const boxesWithHandsT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
const boxesWithHands = boxesWithHandsTensor.arraySync(); const boxesWithHands = boxesWithHandsT.arraySync();
const toDispose = [ const toDispose = [
batchedPrediction, batched,
boxesWithHandsTensor, boxesWithHandsT,
prediction, predictions,
boxes, boxes,
rawBoxes, rawBoxes,
scores, scores,
]; ];
if (boxesWithHands.length === 0) {
toDispose.forEach((tensor) => tensor.dispose());
return null;
}
const hands = []; const hands = [];
for (const boxIndex of boxesWithHands) { for (const boxIndex of boxesWithHands) {
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]); const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]);
const rawPalmLandmarks = tf.slice(prediction, [boxIndex, 5], [1, 14]); const rawPalmLandmarks = tf.slice(predictions, [boxIndex, 5], [1, 14]);
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2])); const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2]));
rawPalmLandmarks.dispose(); rawPalmLandmarks.dispose();
hands.push({ boxes: matchingBox, palmLandmarks }); hands.push({ box: matchingBox, palmLandmarks });
} }
toDispose.forEach((tensor) => tensor.dispose()); toDispose.forEach((tensor) => tensor.dispose());
return hands; return hands;
@ -82,16 +79,16 @@ class HandDetector {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image = tf.tidy(() => input.resizeBilinear([config.inputSize, config.inputSize]).div(127.5).sub(1)); const image = tf.tidy(() => input.resizeBilinear([config.inputSize, config.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoundingBoxes(image, config); const predictions = await this.getBoxes(image, config);
image.dispose(); image.dispose();
if (!predictions || predictions.length === 0) return null; if (!predictions || predictions.length === 0) return null;
const hands = []; const hands = [];
for (const prediction of predictions) { for (const prediction of predictions) {
const boundingBoxes = prediction.boxes.dataSync(); const boundingBoxes = prediction.box.dataSync();
const startPoint = boundingBoxes.slice(0, 2); const startPoint = boundingBoxes.slice(0, 2);
const endPoint = boundingBoxes.slice(2, 4); const endPoint = boundingBoxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose(); prediction.box.dispose();
prediction.palmLandmarks.dispose(); prediction.palmLandmarks.dispose();
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / config.inputSize, inputHeight / config.inputSize])); hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / config.inputSize, inputHeight / config.inputSize]));
} }

60
src/hand/handpipeline.js
View File

@ -30,12 +30,11 @@ const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline { class HandPipeline {
constructor(boundingBoxDetector, meshDetector, inputSize) { constructor(boundingBoxDetector, meshDetector, inputSize) {
this.boundingBoxDetector = boundingBoxDetector; this.boxDetector = boundingBoxDetector;
this.meshDetector = meshDetector; this.meshDetector = meshDetector;
this.inputSize = inputSize; this.inputSize = inputSize;
this.regionsOfInterest = []; this.storedBoxes = [];
this.runsWithoutHandDetector = 0; this.skipped = 0;
this.skipFrames = 0;
this.detectedHands = 0; this.detectedHands = 0;
} }
@ -86,30 +85,24 @@ class HandPipeline {
} }
async estimateHands(image, config) { async estimateHands(image, config) {
this.skipFrames = config.skipFrames; this.skipped++;
// don't need box detection if we have sufficient number of boxes let useFreshBox = false;
let useFreshBox = (this.runsWithoutHandDetector > this.skipFrames) || (this.detectedHands !== this.regionsOfInterest.length); // run new detector every skipFrames
let boundingBoxPredictions; const boxes = (this.skipped > config.skipFrames)
// but every skipFrames check if detect boxes number changed ? await this.boxDetector.estimateHandBounds(image, config) : null;
if (useFreshBox) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config); // if detector result count doesn't match current working set, use it to reset current working set
// if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1 if (boxes && (boxes.length !== this.detectedHands) && (this.detectedHands !== config.maxHands)) {
if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true; // console.log(this.skipped, config.maxHands, this.detectedHands, this.storedBoxes.length, boxes.length);
if (useFreshBox) { this.storedBoxes = [];
this.regionsOfInterest = []; this.detectedHands = 0;
if (!boundingBoxPredictions || boundingBoxPredictions.length === 0) { for (const possible of boxes) this.storedBoxes.push(possible);
this.detectedHands = 0; if (this.storedBoxes.length > 0) useFreshBox = true;
return null; this.skipped = 0;
}
for (const boundingBoxPrediction of boundingBoxPredictions) {
this.regionsOfInterest.push(boundingBoxPrediction);
}
this.runsWithoutHandDetector = 0;
} else {
this.runsWithoutHandDetector++;
} }
const hands = []; const hands = [];
for (const i in this.regionsOfInterest) { // go through working set of boxes
const currentBox = this.regionsOfInterest[i]; for (const i in this.storedBoxes) {
const currentBox = this.storedBoxes[i];
if (!currentBox) continue; if (!currentBox) continue;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]); const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
const palmCenter = box.getBoxCenter(currentBox); const palmCenter = box.getBoxCenter(currentBox);
@ -121,8 +114,7 @@ class HandPipeline {
const handImage = croppedInput.div(255); const handImage = croppedInput.div(255);
croppedInput.dispose(); croppedInput.dispose();
rotatedImage.dispose(); rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage); const [confidence, keypoints] = await this.meshDetector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose(); handImage.dispose();
const confidenceValue = confidence.dataSync()[0]; const confidenceValue = confidence.dataSync()[0];
confidence.dispose(); confidence.dispose();
@ -133,7 +125,7 @@ class HandPipeline {
keypointsReshaped.dispose(); keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords); const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateRegionsOfInterest(nextBoundingBox, i); this.updateStoredBoxes(nextBoundingBox, i);
const result = { const result = {
landmarks: coords, landmarks: coords,
handInViewConfidence: confidenceValue, handInViewConfidence: confidenceValue,
@ -144,7 +136,7 @@ class HandPipeline {
}; };
hands.push(result); hands.push(result);
} else { } else {
this.updateRegionsOfInterest(null, i); this.updateStoredBoxes(null, i);
/* /*
const result = { const result = {
handInViewConfidence: confidenceValue, handInViewConfidence: confidenceValue,
@ -158,7 +150,7 @@ class HandPipeline {
} }
keypoints.dispose(); keypoints.dispose();
} }
this.regionsOfInterest = this.regionsOfInterest.filter((a) => a !== null); this.storedBoxes = this.storedBoxes.filter((a) => a !== null);
this.detectedHands = hands.length; this.detectedHands = hands.length;
return hands; return hands;
} }
@ -172,8 +164,8 @@ class HandPipeline {
return { startPoint, endPoint }; return { startPoint, endPoint };
} }
updateRegionsOfInterest(newBox, i) { updateStoredBoxes(newBox, i) {
const previousBox = this.regionsOfInterest[i]; const previousBox = this.storedBoxes[i];
let iou = 0; let iou = 0;
if (newBox && previousBox && previousBox.startPoint) { if (newBox && previousBox && previousBox.startPoint) {
const [boxStartX, boxStartY] = newBox.startPoint; const [boxStartX, boxStartY] = newBox.startPoint;
@ -189,7 +181,7 @@ class HandPipeline {
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY); const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
iou = intersection / (boxArea + previousBoxArea - intersection); iou = intersection / (boxArea + previousBoxArea - intersection);
} }
this.regionsOfInterest[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox; this.storedBoxes[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox;
} }
} }