update hand detector processing algorithm

pull/193/head
Vladimir Mandic 2021-09-02 08:50:16 -04:00
parent df1a11e911
commit e54304e61a
22 changed files with 175 additions and 180 deletions

View File

@ -11,9 +11,8 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
### **HEAD -> main** 2021/08/31 mandic00@live.com
### **origin/main** 2021/08/31 mandic00@live.com
- simplify canvas handling in nodejs
- full rebuild
### **2.1.5** 2021/08/31 mandic00@live.com

View File

@ -35,7 +35,7 @@ let userConfig = {
/*
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/',
async: false,
cacheSensitivity: 0,
cacheSensitivity: 0.75,
filter: {
enabled: false,
flip: false,
@ -49,11 +49,12 @@ let userConfig = {
},
object: { enabled: false },
gesture: { enabled: true },
hand: { enabled: false },
hand: { enabled: true },
body: { enabled: false },
// body: { enabled: true, modelPath: 'movenet-multipose.json' },
// body: { enabled: true, modelPath: 'posenet.json' },
segmentation: { enabled: false },
/*
*/
};
@ -82,6 +83,7 @@ const ui = {
buffered: true, // should output be buffered between frames
interpolated: true, // should output be interpolated for smoothness between frames
iconSize: '48px', // ui icon sizes
autoPlay: false, // start webcam & detection on load
// internal variables
busy: false, // internal camera busy flag
@ -375,9 +377,9 @@ async function setupCamera() {
canvas.height = video.videoHeight;
ui.menuWidth.input.setAttribute('value', video.videoWidth);
ui.menuHeight.input.setAttribute('value', video.videoHeight);
if (live) video.play();
if (live || ui.autoPlay) video.play();
// eslint-disable-next-line no-use-before-define
if (live && !ui.detectThread) runHumanDetect(video, canvas);
if ((live || ui.autoPlay) && !ui.detectThread) runHumanDetect(video, canvas);
ui.busy = false;
resolve();
};
@ -936,6 +938,10 @@ async function main() {
ui.bench = JSON.parse(params.get('bench'));
log('overriding bench:', ui.bench);
}
if (params.has('play')) {
ui.autoPlay = true;
log('overriding autoplay:', true);
}
if (params.has('draw')) {
ui.drawWarmup = JSON.parse(params.get('draw'));
log('overriding drawWarmup:', ui.drawWarmup);

View File

@ -5,7 +5,7 @@
// import Human from '../dist/human.esm.js';
self.importScripts('../../dist/human.js');
self.importScripts('../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js');
self.importScripts('../../assets/tf-backend-webgpu.es2017.js');
// self.importScripts('../../assets/tf-backend-webgpu.fesm.js');
let human;

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

18
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

18
dist/human.js vendored

File diff suppressed because one or more lines are too long

View File

@ -194,9 +194,9 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 18,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 2,
minConfidence: 0.8,
iouThreshold: 0.2,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
@ -7715,30 +7715,23 @@ var HandDetector = class {
});
}
async getBoxes(input, config3) {
const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const t = {};
t.batched = this.model.predict(input);
t.predictions = tf10.squeeze(t.batched);
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await t.scores.data();
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const nms = await t.nms.array();
const hands = [];
for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
for (const index of nms) {
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
tf10.dispose(predictions);
tf10.dispose(boxes);
for (const tensor2 of Object.keys(t))
tf10.dispose(t[tensor2]);
return hands;
}
async estimateHandBounds(input, config3) {
@ -7926,7 +7919,7 @@ var HandPipeline = class {
tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) {
if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3);
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) {
const slopesXY = [];
const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) {
const points = Finger.getPoints(finger);
const slopeAtXY = [];
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ);
}
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger);
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
}
if (localOptions.drawLabels) {
const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
};
@ -10991,7 +10988,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {};
for (const key of keys) {

View File

@ -195,9 +195,9 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 18,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 2,
minConfidence: 0.8,
iouThreshold: 0.2,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
@ -7716,30 +7716,23 @@ var HandDetector = class {
});
}
async getBoxes(input, config3) {
const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const t = {};
t.batched = this.model.predict(input);
t.predictions = tf10.squeeze(t.batched);
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await t.scores.data();
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const nms = await t.nms.array();
const hands = [];
for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
for (const index of nms) {
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
tf10.dispose(predictions);
tf10.dispose(boxes);
for (const tensor2 of Object.keys(t))
tf10.dispose(t[tensor2]);
return hands;
}
async estimateHandBounds(input, config3) {
@ -7927,7 +7920,7 @@ var HandPipeline = class {
tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) {
if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3);
@ -8181,6 +8174,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) {
const slopesXY = [];
const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) {
const points = Finger.getPoints(finger);
const slopeAtXY = [];
@ -8197,8 +8194,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ);
}
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger);
@ -10793,6 +10788,8 @@ async function hand2(inCanvas2, result, drawOptions) {
}
if (localOptions.drawLabels) {
const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
};
@ -10992,7 +10989,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {};
for (const key of keys) {

55
dist/human.node.js vendored
View File

@ -194,9 +194,9 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 18,
minConfidence: 0.1,
iouThreshold: 0.1,
maxDetected: 2,
minConfidence: 0.8,
iouThreshold: 0.2,
maxDetected: 1,
landmarks: true,
detector: {
modelPath: "handdetect.json"
@ -7715,30 +7715,23 @@ var HandDetector = class {
});
}
async getBoxes(input, config3) {
const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = await filteredT.array();
tf10.dispose(scoresT);
tf10.dispose(filteredT);
const t = {};
t.batched = this.model.predict(input);
t.predictions = tf10.squeeze(t.batched);
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await t.scores.data();
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const nms = await t.nms.array();
const hands = [];
for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
for (const index of nms) {
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
tf10.dispose(predictions);
tf10.dispose(boxes);
for (const tensor2 of Object.keys(t))
tf10.dispose(t[tensor2]);
return hands;
}
async estimateHandBounds(input, config3) {
@ -7926,7 +7919,7 @@ var HandPipeline = class {
tf11.dispose(handImage);
const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) {
if (confidence >= config3.hand.minConfidence / 4) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
tf11.dispose(keypoints3);
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
function estimate(landmarks) {
const slopesXY = [];
const slopesYZ = [];
const fingerCurls = [];
const fingerDirections = [];
if (!landmarks)
return { curls: fingerCurls, directions: fingerDirections };
for (const finger of Finger.all) {
const points = Finger.getPoints(finger);
const slopeAtXY = [];
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
slopesXY.push(slopeAtXY);
slopesYZ.push(slopeAtYZ);
}
const fingerCurls = [];
const fingerDirections = [];
for (const finger of Finger.all) {
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
const fingerPointsAt = Finger.getPoints(finger);
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
}
if (localOptions.drawLabels) {
const addHandLabel = (part, title) => {
if (!part)
return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
};
@ -10991,7 +10988,7 @@ function calc(newResult) {
for (let i = 0; i < newResult.hand.length; i++) {
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
const keys = Object.keys(newResult.hand[i].annotations);
const annotations3 = {};
for (const key of keys) {

4
dist/tfjs.esm.js vendored
View File

@ -2173,9 +2173,9 @@ var require_seedrandom4 = __commonJS({
}
});
// (disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js
// (disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js
var require_string_decoder = __commonJS({
"(disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js"() {
"(disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js"() {
}
});

File diff suppressed because one or more lines are too long

View File

@ -66,7 +66,7 @@
"@tensorflow/tfjs-layers": "^3.9.0",
"@tensorflow/tfjs-node": "^3.9.0",
"@tensorflow/tfjs-node-gpu": "^3.9.0",
"@types/node": "^16.7.8",
"@types/node": "^16.7.10",
"@typescript-eslint/eslint-plugin": "^4.30.0",
"@typescript-eslint/parser": "^4.30.0",
"@vladmandic/pilogger": "^0.2.18",

View File

@ -1,22 +1,22 @@
2021-08-31 18:22:47 INFO:  @vladmandic/human version 2.1.5
2021-08-31 18:22:47 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-31 18:22:47 INFO:  Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"}
2021-08-31 18:22:47 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-31 18:22:47 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-31 18:22:47 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: node type: node: {"imports":47,"importBytes":456300,"outputBytes":396489,"outputFiles":"dist/human.node.js"}
2021-08-31 18:22:47 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: nodeGPU type: node: {"imports":47,"importBytes":456308,"outputBytes":396493,"outputFiles":"dist/human.node-gpu.js"}
2021-08-31 18:22:47 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: nodeWASM type: node: {"imports":47,"importBytes":456375,"outputBytes":396565,"outputFiles":"dist/human.node-wasm.js"}
2021-08-31 18:22:47 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:47 STATE: target: browserNoBundle type: esm: {"imports":47,"importBytes":456239,"outputBytes":255270,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-31 18:22:48 STATE: target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"}
2021-08-31 18:22:48 STATE: target: browserBundle type: iife: {"imports":47,"importBytes":2798929,"outputBytes":1391500,"outputFiles":"dist/human.js"}
2021-08-31 18:22:49 STATE: target: browserBundle type: esm: {"imports":47,"importBytes":2798929,"outputBytes":1391492,"outputFiles":"dist/human.esm.js"}
2021-08-31 18:22:49 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-31 18:23:12 INFO:  Linter complete: files: 84 errors: 0 warnings: 0
2021-08-31 18:23:12 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-31 18:23:12 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-31 18:23:26 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-31 18:23:40 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1
2021-09-02 08:49:02 INFO:  @vladmandic/human version 2.1.5
2021-09-02 08:49:02 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-09-02 08:49:02 INFO:  Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"}
2021-09-02 08:49:02 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-09-02 08:49:02 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-09-02 08:49:02 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-09-02 08:49:02 STATE: target: node type: node: {"imports":47,"importBytes":456773,"outputBytes":396371,"outputFiles":"dist/human.node.js"}
2021-09-02 08:49:02 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-09-02 08:49:02 STATE: target: nodeGPU type: node: {"imports":47,"importBytes":456781,"outputBytes":396375,"outputFiles":"dist/human.node-gpu.js"}
2021-09-02 08:49:02 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-09-02 08:49:02 STATE: target: nodeWASM type: node: {"imports":47,"importBytes":456848,"outputBytes":396447,"outputFiles":"dist/human.node-wasm.js"}
2021-09-02 08:49:02 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-09-02 08:49:02 STATE: target: browserNoBundle type: esm: {"imports":47,"importBytes":456712,"outputBytes":255375,"outputFiles":"dist/human.esm-nobundle.js"}
2021-09-02 08:49:03 STATE: target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"}
2021-09-02 08:49:03 STATE: target: browserBundle type: iife: {"imports":47,"importBytes":2799402,"outputBytes":1391653,"outputFiles":"dist/human.js"}
2021-09-02 08:49:04 STATE: target: browserBundle type: esm: {"imports":47,"importBytes":2799402,"outputBytes":1391645,"outputFiles":"dist/human.esm.js"}
2021-09-02 08:49:04 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-09-02 08:49:28 INFO:  Linter complete: files: 84 errors: 0 warnings: 0
2021-09-02 08:49:28 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-09-02 08:49:28 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-09-02 08:49:42 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-09-02 08:49:56 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1

View File

@ -331,9 +331,9 @@ const config: Config = {
// e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated hand skeleton analysis as the hand probably
// hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
maxDetected: 2, // maximum number of hands detected in the input
minConfidence: 0.8, // threshold for discarding a prediction
iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed
maxDetected: 1, // maximum number of hands detected in the input
// should be set to the minimum number for performance
landmarks: true, // detect hand landmarks or just hand boundary box
detector: {

View File

@ -407,6 +407,7 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
}
if (localOptions.drawLabels) {
const addHandLabel = (part, title) => {
if (!part) return;
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[part.length - 1][2])}, ${127.5 - (2 * part[part.length - 1][2])}, 255, 0.5)` : localOptions.color;
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
};

View File

@ -167,6 +167,11 @@ export function estimate(landmarks) {
// step 1: calculate slopes
const slopesXY: Array<number[]> = [];
const slopesYZ: Array<number[]> = [];
const fingerCurls: Array<number> = [];
const fingerDirections: Array<number> = [];
if (!landmarks) return { curls: fingerCurls, directions: fingerDirections };
// step 1: calculate slopes
for (const finger of Finger.all) {
const points = Finger.getPoints(finger);
const slopeAtXY: Array<number> = [];
@ -186,8 +191,6 @@ export function estimate(landmarks) {
}
// step 2: calculate orientations
const fingerCurls: Array<number> = [];
const fingerDirections: Array<number> = [];
for (const finger of Finger.all) {
// start finger predictions from palm - except for thumb
const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;

View File

@ -40,31 +40,23 @@ export class HandDetector {
}
async getBoxes(input, config) {
const batched = this.model.predict(input) as Tensor;
const predictions = tf.squeeze(batched);
tf.dispose(batched);
const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1]))));
const scores = await scoresT.data();
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf.dispose(rawBoxes);
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const filtered = await filteredT.array();
tf.dispose(scoresT);
tf.dispose(filteredT);
const t: Record<string, Tensor> = {};
t.batched = this.model.predict(input) as Tensor;
t.predictions = tf.squeeze(t.batched);
t.scores = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(t.predictions, [0, 0], [-1, 1]))));
const scores = await t.scores.data();
t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const nms = await t.nms.array() as Array<number>;
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
for (const index of filtered) {
if (scores[index] >= config.hand.minConfidence) {
const matchingBox = tf.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
tf.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
}
for (const index of nms) {
const palmBox = tf.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(tf.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
// console.log('handdetector:getBoxes', nms.length, index, scores[index], config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence, palmBox.dataSync());
}
tf.dispose(predictions);
tf.dispose(boxes);
for (const tensor of Object.keys(t)) tf.dispose(t[tensor]); // dispose all
return hands;
}

View File

@ -85,7 +85,7 @@ export class HandPipeline {
// run new detector every skipFrames unless we only want box to start with
let boxes;
// console.log(this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame);
// console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image, config);
this.skipped = 0;
@ -120,7 +120,7 @@ export class HandPipeline {
tf.dispose(handImage);
const confidence = (await confidenceT.data())[0];
tf.dispose(confidenceT);
if (confidence >= config.hand.minConfidence) {
if (confidence >= config.hand.minConfidence / 4) {
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
tf.dispose(keypoints);
@ -135,6 +135,7 @@ export class HandPipeline {
};
hands.push(result);
} else {
// console.log('handpipeline:estimateHands low', confidence);
this.storedBoxes[i] = null;
}
tf.dispose(keypoints);

View File

@ -148,8 +148,8 @@ export class Human {
* @param userConfig: {@link Config}
*/
constructor(userConfig?: Config | Record<string, unknown>) {
Human.version = app.version;
Object.defineProperty(this, 'version', { value: app.version });
Human.version = app.version; // expose version property on instance of class
Object.defineProperty(this, 'version', { value: app.version }); // expose version property directly on class itself
defaults.wasmPath = `https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tf.version_core}/dist/`;
this.config = mergeDeep(defaults, userConfig || {});
this.tf = tf;
@ -427,6 +427,7 @@ export class Human {
const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff);
// if difference is above 10x threshold, don't use last value to force reset cache for significant change of scenes or images
this.#lastCacheDiff = diff > 10 * this.config.cacheSensitivity ? 0 : diff;
// console.log('skipFrame', skipFrame, this.config.cacheSensitivity, diff);
return skipFrame;
}

View File

@ -59,9 +59,10 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
const keypoints = newResult.hand[i].keypoints // update landmarks
const keypoints = newResult.hand[i].keypoints ? newResult.hand[i].keypoints // update landmarks
.map((landmark, j) => landmark
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number]);
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number])
: [];
const keys = Object.keys(newResult.hand[i].annotations); // update annotations
const annotations = {};
for (const key of keys) {