ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update hand detector processing algorithm

pull/193/head
Vladimir Mandic 2021-09-02 08:50:16 -04:00
parent df1a11e911
commit e54304e61a
22 changed files with 175 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -11,9 +11,8 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/08/31 mandic00@live.com ### **HEAD -> main** 2021/08/31 mandic00@live.com
- simplify canvas handling in nodejs
### **origin/main** 2021/08/31 mandic00@live.com - full rebuild
### **2.1.5** 2021/08/31 mandic00@live.com ### **2.1.5** 2021/08/31 mandic00@live.com

14
demo/index.js
View File

@ -35,7 +35,7 @@ let userConfig = {
/* /*
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/',
async: false, async: false,
cacheSensitivity: 0, cacheSensitivity: 0.75,
filter: { filter: {
enabled: false, enabled: false,
flip: false, flip: false,
@ -49,11 +49,12 @@ let userConfig = {
}, },
object: { enabled: false }, object: { enabled: false },
gesture: { enabled: true }, gesture: { enabled: true },
hand: { enabled: false }, hand: { enabled: true },
body: { enabled: false }, body: { enabled: false },
// body: { enabled: true, modelPath: 'movenet-multipose.json' }, // body: { enabled: true, modelPath: 'movenet-multipose.json' },
// body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'posenet.json' },
segmentation: { enabled: false }, segmentation: { enabled: false },
/*
*/ */
}; };
@ -82,6 +83,7 @@ const ui = {
buffered: true, // should output be buffered between frames buffered: true, // should output be buffered between frames
interpolated: true, // should output be interpolated for smoothness between frames interpolated: true, // should output be interpolated for smoothness between frames
iconSize: '48px', // ui icon sizes iconSize: '48px', // ui icon sizes
autoPlay: false, // start webcam & detection on load
// internal variables // internal variables
busy: false, // internal camera busy flag busy: false, // internal camera busy flag
@ -375,9 +377,9 @@ async function setupCamera() {
canvas.height = video.videoHeight; canvas.height = video.videoHeight;
ui.menuWidth.input.setAttribute('value', video.videoWidth); ui.menuWidth.input.setAttribute('value', video.videoWidth);
ui.menuHeight.input.setAttribute('value', video.videoHeight); ui.menuHeight.input.setAttribute('value', video.videoHeight);
if (live) video.play(); if (live || ui.autoPlay) video.play();
// eslint-disable-next-line no-use-before-define // eslint-disable-next-line no-use-before-define
if (live && !ui.detectThread) runHumanDetect(video, canvas); if ((live || ui.autoPlay) && !ui.detectThread) runHumanDetect(video, canvas);
ui.busy = false; ui.busy = false;
resolve(); resolve();
}; };
@ -936,6 +938,10 @@ async function main() {
ui.bench = JSON.parse(params.get('bench')); ui.bench = JSON.parse(params.get('bench'));
log('overriding bench:', ui.bench); log('overriding bench:', ui.bench);
} }
if (params.has('play')) {
ui.autoPlay = true;
log('overriding autoplay:', true);
}
if (params.has('draw')) { if (params.has('draw')) {
ui.drawWarmup = JSON.parse(params.get('draw')); ui.drawWarmup = JSON.parse(params.get('draw'));
log('overriding drawWarmup:', ui.drawWarmup); log('overriding drawWarmup:', ui.drawWarmup);

2
demo/webgpu/worker.js
View File

@ -5,7 +5,7 @@
// import Human from '../dist/human.esm.js'; // import Human from '../dist/human.esm.js';
self.importScripts('../../dist/human.js'); self.importScripts('../../dist/human.js');
self.importScripts('../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js'); self.importScripts('../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js');
self.importScripts('../../assets/tf-backend-webgpu.es2017.js'); // self.importScripts('../../assets/tf-backend-webgpu.fesm.js');
let human; let human;

4
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

18
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

18
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

55
dist/human.node-gpu.js vendored
View File

@ -194,9 +194,9 @@ var config = {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 18, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.8,
iouThreshold: 0.1, iouThreshold: 0.2,
maxDetected: 2, maxDetected: 1,
landmarks: true, landmarks: true,
detector: { detector: {
modelPath: "handdetect.json" modelPath: "handdetect.json"
@ -7715,30 +7715,23 @@ var HandDetector = class {
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const t = {};
const predictions = tf10.squeeze(batched); t.batched = this.model.predict(input);
tf10.dispose(batched); t.predictions = tf10.squeeze(t.batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])))); t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data(); const scores = await t.scores.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); t.norm = this.normalizeBoxes(t.boxes);
tf10.dispose(rawBoxes); t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const nms = await t.nms.array();
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of nms) {
if (scores[index] >= config3.hand.minConfidence) { const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
} }
tf10.dispose(predictions); for (const tensor2 of Object.keys(t))
tf10.dispose(boxes); tf10.dispose(t[tensor2]);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
@ -7926,7 +7919,7 @@ var HandPipeline = class {
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array(); const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3); tf11.dispose(keypoints3);
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) { function estimate(landmarks) {
const slopesXY = []; const slopesXY = [];
const slopesYZ = []; const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) { for (const finger of Finger.all) {
const points = Finger.getPoints(finger); const points = Finger.getPoints(finger);
const slopeAtXY = []; const slopeAtXY = [];
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY); slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ); slopesYZ.push(slopeAtYZ);
} }
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) { for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0; const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger); const fingerPointsAt = Finger.getPoints(finger);
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
const addHandLabel = (part, title) => { const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
}; };
@ -10991,7 +10988,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) { for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor); const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor); const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)); const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations); const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {}; const annotations3 = {};
for (const key of keys) { for (const key of keys) {

55
dist/human.node-wasm.js vendored
View File

@ -195,9 +195,9 @@ var config = {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 18, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.8,
iouThreshold: 0.1, iouThreshold: 0.2,
maxDetected: 2, maxDetected: 1,
landmarks: true, landmarks: true,
detector: { detector: {
modelPath: "handdetect.json" modelPath: "handdetect.json"
@ -7716,30 +7716,23 @@ var HandDetector = class {
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const t = {};
const predictions = tf10.squeeze(batched); t.batched = this.model.predict(input);
tf10.dispose(batched); t.predictions = tf10.squeeze(t.batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])))); t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data(); const scores = await t.scores.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); t.norm = this.normalizeBoxes(t.boxes);
tf10.dispose(rawBoxes); t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const nms = await t.nms.array();
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of nms) {
if (scores[index] >= config3.hand.minConfidence) { const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
} }
tf10.dispose(predictions); for (const tensor2 of Object.keys(t))
tf10.dispose(boxes); tf10.dispose(t[tensor2]);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
@ -7927,7 +7920,7 @@ var HandPipeline = class {
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array(); const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3); tf11.dispose(keypoints3);
@ -8181,6 +8174,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) { function estimate(landmarks) {
const slopesXY = []; const slopesXY = [];
const slopesYZ = []; const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) { for (const finger of Finger.all) {
const points = Finger.getPoints(finger); const points = Finger.getPoints(finger);
const slopeAtXY = []; const slopeAtXY = [];
@ -8197,8 +8194,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY); slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ); slopesYZ.push(slopeAtYZ);
} }
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) { for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0; const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger); const fingerPointsAt = Finger.getPoints(finger);
@ -10793,6 +10788,8 @@ async function hand2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
const addHandLabel = (part, title) => { const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
}; };
@ -10992,7 +10989,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) { for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor); const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor); const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)); const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations); const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {}; const annotations3 = {};
for (const key of keys) { for (const key of keys) {

55
dist/human.node.js vendored
View File

@ -194,9 +194,9 @@ var config = {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 18, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.8,
iouThreshold: 0.1, iouThreshold: 0.2,
maxDetected: 2, maxDetected: 1,
landmarks: true, landmarks: true,
detector: { detector: {
modelPath: "handdetect.json" modelPath: "handdetect.json"
@ -7715,30 +7715,23 @@ var HandDetector = class {
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const t = {};
const predictions = tf10.squeeze(batched); t.batched = this.model.predict(input);
tf10.dispose(batched); t.predictions = tf10.squeeze(t.batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])))); t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data(); const scores = await t.scores.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); t.norm = this.normalizeBoxes(t.boxes);
tf10.dispose(rawBoxes); t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const nms = await t.nms.array();
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of nms) {
if (scores[index] >= config3.hand.minConfidence) { const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
} }
tf10.dispose(predictions); for (const tensor2 of Object.keys(t))
tf10.dispose(boxes); tf10.dispose(t[tensor2]);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
@ -7926,7 +7919,7 @@ var HandPipeline = class {
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array(); const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3); tf11.dispose(keypoints3);
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) { function estimate(landmarks) {
const slopesXY = []; const slopesXY = [];
const slopesYZ = []; const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) { for (const finger of Finger.all) {
const points = Finger.getPoints(finger); const points = Finger.getPoints(finger);
const slopeAtXY = []; const slopeAtXY = [];
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY); slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ); slopesYZ.push(slopeAtYZ);
} }
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) { for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0; const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger); const fingerPointsAt = Finger.getPoints(finger);
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
const addHandLabel = (part, title) => { const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
}; };
@ -10991,7 +10988,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) { for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor); const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor); const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)); const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations); const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {}; const annotations3 = {};
for (const key of keys) { for (const key of keys) {

4
dist/tfjs.esm.js vendored
View File

@ -2173,9 +2173,9 @@ var require_seedrandom4 = __commonJS({
} }
}); });
// (disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js // (disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js
var require_string_decoder = __commonJS({ var require_string_decoder = __commonJS({
"(disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js"() { "(disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js"() {
} }
}); });

2
dist/tfjs.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
package.json
View File

@ -66,7 +66,7 @@
"@tensorflow/tfjs-layers": "^3.9.0", "@tensorflow/tfjs-layers": "^3.9.0",
"@tensorflow/tfjs-node": "^3.9.0", "@tensorflow/tfjs-node": "^3.9.0",
"@tensorflow/tfjs-node-gpu": "^3.9.0", "@tensorflow/tfjs-node-gpu": "^3.9.0",
"@types/node": "^16.7.8", "@types/node": "^16.7.10",
"@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/eslint-plugin": "^4.30.0",
"@typescript-eslint/parser": "^4.30.0", "@typescript-eslint/parser": "^4.30.0",
"@vladmandic/pilogger": "^0.2.18", "@vladmandic/pilogger": "^0.2.18",

44
server/build.log
View File

@ -1,22 +1,22 @@
2021-08-31 18:22:47 INFO:  @vladmandic/human version 2.1.5 2021-09-02 08:49:02 INFO:  @vladmandic/human version 2.1.5
2021-08-31 18:22:47 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0 2021-09-02 08:49:02 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-31 18:22:47 INFO:  Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"} 2021-09-02 08:49:02 INFO:  Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"}
2021-08-31 18:22:47 INFO:  Clean: ["dist/*","types/*","typedoc/*"] 2021-09-02 08:49:02 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-31 18:22:47 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 2021-09-02 08:49:02 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-31 18:22:47 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"} 2021-09-02 08:49:02 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: node type: node: {"imports":47,"importBytes":456300,"outputBytes":396489,"outputFiles":"dist/human.node.js"} 2021-09-02 08:49:02 STATE: target: node type: node: {"imports":47,"importBytes":456773,"outputBytes":396371,"outputFiles":"dist/human.node.js"}
2021-08-31 18:22:47 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"} 2021-09-02 08:49:02 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: nodeGPU type: node: {"imports":47,"importBytes":456308,"outputBytes":396493,"outputFiles":"dist/human.node-gpu.js"} 2021-09-02 08:49:02 STATE: target: nodeGPU type: node: {"imports":47,"importBytes":456781,"outputBytes":396375,"outputFiles":"dist/human.node-gpu.js"}
2021-08-31 18:22:47 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"} 2021-09-02 08:49:02 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: nodeWASM type: node: {"imports":47,"importBytes":456375,"outputBytes":396565,"outputFiles":"dist/human.node-wasm.js"} 2021-09-02 08:49:02 STATE: target: nodeWASM type: node: {"imports":47,"importBytes":456848,"outputBytes":396447,"outputFiles":"dist/human.node-wasm.js"}
2021-08-31 18:22:47 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"} 2021-09-02 08:49:02 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: browserNoBundle type: esm: {"imports":47,"importBytes":456239,"outputBytes":255270,"outputFiles":"dist/human.esm-nobundle.js"} 2021-09-02 08:49:02 STATE: target: browserNoBundle type: esm: {"imports":47,"importBytes":456712,"outputBytes":255375,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-31 18:22:48 STATE: target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"} 2021-09-02 08:49:03 STATE: target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:48 STATE: target: browserBundle type: iife: {"imports":47,"importBytes":2798929,"outputBytes":1391500,"outputFiles":"dist/human.js"} 2021-09-02 08:49:03 STATE: target: browserBundle type: iife: {"imports":47,"importBytes":2799402,"outputBytes":1391653,"outputFiles":"dist/human.js"}
2021-08-31 18:22:49 STATE: target: browserBundle type: esm: {"imports":47,"importBytes":2798929,"outputBytes":1391492,"outputFiles":"dist/human.esm.js"} 2021-09-02 08:49:04 STATE: target: browserBundle type: esm: {"imports":47,"importBytes":2799402,"outputBytes":1391645,"outputFiles":"dist/human.esm.js"}
2021-08-31 18:22:49 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"] 2021-09-02 08:49:04 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-31 18:23:12 INFO:  Linter complete: files: 84 errors: 0 warnings: 0 2021-09-02 08:49:28 INFO:  Linter complete: files: 84 errors: 0 warnings: 0
2021-08-31 18:23:12 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"] 2021-09-02 08:49:28 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-31 18:23:12 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"] 2021-09-02 08:49:28 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-31 18:23:26 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"] 2021-09-02 08:49:42 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-31 18:23:40 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 2021-09-02 08:49:56 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1

6
src/config.ts
View File

@ -331,9 +331,9 @@ const config: Config = {
// e.g., if model is running st 25 FPS, we can re-use existing bounding // e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated hand skeleton analysis as the hand probably // box for updated hand skeleton analysis as the hand probably
// hasn't moved much in short time (10 * 1/25 = 0.25 sec) // hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.8, // threshold for discarding a prediction
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed
maxDetected: 2, // maximum number of hands detected in the input maxDetected: 1, // maximum number of hands detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
landmarks: true, // detect hand landmarks or just hand boundary box landmarks: true, // detect hand landmarks or just hand boundary box
detector: { detector: {

1
src/draw/draw.ts
View File

@ -407,6 +407,7 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
} }
if (localOptions.drawLabels) { if (localOptions.drawLabels) {
const addHandLabel = (part, title) => { const addHandLabel = (part, title) => {
if (!part) return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[part.length - 1][2])}, ${127.5 - (2 * part[part.length - 1][2])}, 255, 0.5)` : localOptions.color; ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[part.length - 1][2])}, ${127.5 - (2 * part[part.length - 1][2])}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
}; };

7
src/fingerpose/estimator.ts
View File

@ -167,6 +167,11 @@ export function estimate(landmarks) {
// step 1: calculate slopes // step 1: calculate slopes
const slopesXY: Array<number[]> = []; const slopesXY: Array<number[]> = [];
const slopesYZ: Array<number[]> = []; const slopesYZ: Array<number[]> = [];
const fingerCurls: Array<number> = [];
const fingerDirections: Array<number> = [];
if (!landmarks) return { curls: fingerCurls, directions: fingerDirections };
// step 1: calculate slopes
for (const finger of Finger.all) { for (const finger of Finger.all) {
const points = Finger.getPoints(finger); const points = Finger.getPoints(finger);
const slopeAtXY: Array<number> = []; const slopeAtXY: Array<number> = [];
@ -186,8 +191,6 @@ export function estimate(landmarks) {
} }
// step 2: calculate orientations // step 2: calculate orientations
const fingerCurls: Array<number> = [];
const fingerDirections: Array<number> = [];
for (const finger of Finger.all) { for (const finger of Finger.all) {
// start finger predictions from palm - except for thumb // start finger predictions from palm - except for thumb
const pointIndexAt = (finger === Finger.thumb) ? 1 : 0; const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;

38
src/handpose/handdetector.ts
View File

@ -40,31 +40,23 @@ export class HandDetector {
} }
async getBoxes(input, config) { async getBoxes(input, config) {
const batched = this.model.predict(input) as Tensor; const t: Record<string, Tensor> = {};
const predictions = tf.squeeze(batched); t.batched = this.model.predict(input) as Tensor;
tf.dispose(batched); t.predictions = tf.squeeze(t.batched);
const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])))); t.scores = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data(); const scores = await t.scores.data();
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); t.norm = this.normalizeBoxes(t.boxes);
tf.dispose(rawBoxes); t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); const nms = await t.nms.array() as Array<number>;
const filtered = await filteredT.array();
tf.dispose(scoresT);
tf.dispose(filteredT);
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = []; const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
for (const index of filtered) { for (const index of nms) {
if (scores[index] >= config.hand.minConfidence) { const palmBox = tf.slice(t.norm, [index, 0], [1, -1]);
const matchingBox = tf.slice(boxes, [index, 0], [1, -1]); const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(tf.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
const rawPalmLandmarks = tf.slice(predictions, [index, 5], [1, 14]); hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2])); // console.log('handdetector:getBoxes', nms.length, index, scores[index], config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence, palmBox.dataSync());
tf.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
} }
tf.dispose(predictions); for (const tensor of Object.keys(t)) tf.dispose(t[tensor]); // dispose all
tf.dispose(boxes);
return hands; return hands;
} }

5
src/handpose/handpipeline.ts
View File

@ -85,7 +85,7 @@ export class HandPipeline {
// run new detector every skipFrames unless we only want box to start with // run new detector every skipFrames unless we only want box to start with
let boxes; let boxes;
// console.log(this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) { if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image, config); boxes = await this.handDetector.estimateHandBounds(image, config);
this.skipped = 0; this.skipped = 0;
@ -120,7 +120,7 @@ export class HandPipeline {
tf.dispose(handImage); tf.dispose(handImage);
const confidence = (await confidenceT.data())[0]; const confidence = (await confidenceT.data())[0];
tf.dispose(confidenceT); tf.dispose(confidenceT);
if (confidence >= config.hand.minConfidence) { if (confidence >= config.hand.minConfidence / 4) {
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]); const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array(); const rawCoords = await keypointsReshaped.array();
tf.dispose(keypoints); tf.dispose(keypoints);
@ -135,6 +135,7 @@ export class HandPipeline {
}; };
hands.push(result); hands.push(result);
} else { } else {
// console.log('handpipeline:estimateHands low', confidence);
this.storedBoxes[i] = null; this.storedBoxes[i] = null;
} }
tf.dispose(keypoints); tf.dispose(keypoints);

5
src/human.ts
View File

@ -148,8 +148,8 @@ export class Human {
* @param userConfig: {@link Config} * @param userConfig: {@link Config}
*/ */
constructor(userConfig?: Config | Record<string, unknown>) { constructor(userConfig?: Config | Record<string, unknown>) {
Human.version = app.version; Human.version = app.version; // expose version property on instance of class
Object.defineProperty(this, 'version', { value: app.version }); Object.defineProperty(this, 'version', { value: app.version }); // expose version property directly on class itself
defaults.wasmPath = `https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tf.version_core}/dist/`; defaults.wasmPath = `https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tf.version_core}/dist/`;
this.config = mergeDeep(defaults, userConfig || {}); this.config = mergeDeep(defaults, userConfig || {});
this.tf = tf; this.tf = tf;
@ -427,6 +427,7 @@ export class Human {
const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff); const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff);
// if difference is above 10x threshold, don't use last value to force reset cache for significant change of scenes or images // if difference is above 10x threshold, don't use last value to force reset cache for significant change of scenes or images
this.#lastCacheDiff = diff > 10 * this.config.cacheSensitivity ? 0 : diff; this.#lastCacheDiff = diff > 10 * this.config.cacheSensitivity ? 0 : diff;
// console.log('skipFrame', skipFrame, this.config.cacheSensitivity, diff);
return skipFrame; return skipFrame;
} }

5
src/interpolate.ts
View File

@ -59,9 +59,10 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
const boxRaw = (newResult.hand[i].boxRaw // update boxRaw const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number]; .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
const keypoints = newResult.hand[i].keypoints // update landmarks const keypoints = newResult.hand[i].keypoints ? newResult.hand[i].keypoints // update landmarks
.map((landmark, j) => landmark .map((landmark, j) => landmark
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number]); .map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number])
: [];
const keys = Object.keys(newResult.hand[i].annotations); // update annotations const keys = Object.keys(newResult.hand[i].annotations); // update annotations
const annotations = {}; const annotations = {};
for (const key of keys) { for (const key of keys) {