mirror of https://github.com/vladmandic/human
update hand detector processing algorithm
parent
df1a11e911
commit
e54304e61a
|
@ -11,9 +11,8 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
### **HEAD -> main** 2021/08/31 mandic00@live.com
|
||||
|
||||
|
||||
### **origin/main** 2021/08/31 mandic00@live.com
|
||||
|
||||
- simplify canvas handling in nodejs
|
||||
- full rebuild
|
||||
|
||||
### **2.1.5** 2021/08/31 mandic00@live.com
|
||||
|
||||
|
|
|
@ -35,7 +35,7 @@ let userConfig = {
|
|||
/*
|
||||
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/',
|
||||
async: false,
|
||||
cacheSensitivity: 0,
|
||||
cacheSensitivity: 0.75,
|
||||
filter: {
|
||||
enabled: false,
|
||||
flip: false,
|
||||
|
@ -49,11 +49,12 @@ let userConfig = {
|
|||
},
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
hand: { enabled: false },
|
||||
hand: { enabled: true },
|
||||
body: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'movenet-multipose.json' },
|
||||
// body: { enabled: true, modelPath: 'posenet.json' },
|
||||
segmentation: { enabled: false },
|
||||
/*
|
||||
*/
|
||||
};
|
||||
|
||||
|
@ -82,6 +83,7 @@ const ui = {
|
|||
buffered: true, // should output be buffered between frames
|
||||
interpolated: true, // should output be interpolated for smoothness between frames
|
||||
iconSize: '48px', // ui icon sizes
|
||||
autoPlay: false, // start webcam & detection on load
|
||||
|
||||
// internal variables
|
||||
busy: false, // internal camera busy flag
|
||||
|
@ -375,9 +377,9 @@ async function setupCamera() {
|
|||
canvas.height = video.videoHeight;
|
||||
ui.menuWidth.input.setAttribute('value', video.videoWidth);
|
||||
ui.menuHeight.input.setAttribute('value', video.videoHeight);
|
||||
if (live) video.play();
|
||||
if (live || ui.autoPlay) video.play();
|
||||
// eslint-disable-next-line no-use-before-define
|
||||
if (live && !ui.detectThread) runHumanDetect(video, canvas);
|
||||
if ((live || ui.autoPlay) && !ui.detectThread) runHumanDetect(video, canvas);
|
||||
ui.busy = false;
|
||||
resolve();
|
||||
};
|
||||
|
@ -936,6 +938,10 @@ async function main() {
|
|||
ui.bench = JSON.parse(params.get('bench'));
|
||||
log('overriding bench:', ui.bench);
|
||||
}
|
||||
if (params.has('play')) {
|
||||
ui.autoPlay = true;
|
||||
log('overriding autoplay:', true);
|
||||
}
|
||||
if (params.has('draw')) {
|
||||
ui.drawWarmup = JSON.parse(params.get('draw'));
|
||||
log('overriding drawWarmup:', ui.drawWarmup);
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
// import Human from '../dist/human.esm.js';
|
||||
self.importScripts('../../dist/human.js');
|
||||
self.importScripts('../../node_modules/@tensorflow/tfjs-core/dist/tf-core.es2017.js');
|
||||
self.importScripts('../../assets/tf-backend-webgpu.es2017.js');
|
||||
// self.importScripts('../../assets/tf-backend-webgpu.fesm.js');
|
||||
|
||||
let human;
|
||||
|
||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -194,9 +194,9 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 18,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: 1,
|
||||
landmarks: true,
|
||||
detector: {
|
||||
modelPath: "handdetect.json"
|
||||
|
@ -7715,30 +7715,23 @@ var HandDetector = class {
|
|||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const batched = this.model.predict(input);
|
||||
const predictions = tf10.squeeze(batched);
|
||||
tf10.dispose(batched);
|
||||
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await scoresT.data();
|
||||
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
|
||||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
tf10.dispose(rawBoxes);
|
||||
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const filtered = await filteredT.array();
|
||||
tf10.dispose(scoresT);
|
||||
tf10.dispose(filteredT);
|
||||
const t = {};
|
||||
t.batched = this.model.predict(input);
|
||||
t.predictions = tf10.squeeze(t.batched);
|
||||
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of filtered) {
|
||||
if (scores[index] >= config3.hand.minConfidence) {
|
||||
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
|
||||
tf10.dispose(rawPalmLandmarks);
|
||||
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
|
||||
for (const index of nms) {
|
||||
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
}
|
||||
tf10.dispose(predictions);
|
||||
tf10.dispose(boxes);
|
||||
for (const tensor2 of Object.keys(t))
|
||||
tf10.dispose(t[tensor2]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
|
@ -7926,7 +7919,7 @@ var HandPipeline = class {
|
|||
tf11.dispose(handImage);
|
||||
const confidence = (await confidenceT.data())[0];
|
||||
tf11.dispose(confidenceT);
|
||||
if (confidence >= config3.hand.minConfidence) {
|
||||
if (confidence >= config3.hand.minConfidence / 4) {
|
||||
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
|
||||
const rawCoords = await keypointsReshaped.array();
|
||||
tf11.dispose(keypoints3);
|
||||
|
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
|
|||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
if (!landmarks)
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
|
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
|
|||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
|
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
if (localOptions.drawLabels) {
|
||||
const addHandLabel = (part, title) => {
|
||||
if (!part)
|
||||
return;
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
|
@ -10991,7 +10988,7 @@ function calc(newResult) {
|
|||
for (let i = 0; i < newResult.hand.length; i++) {
|
||||
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
|
||||
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
|
||||
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
|
||||
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
|
||||
const keys = Object.keys(newResult.hand[i].annotations);
|
||||
const annotations3 = {};
|
||||
for (const key of keys) {
|
||||
|
|
|
@ -195,9 +195,9 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 18,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: 1,
|
||||
landmarks: true,
|
||||
detector: {
|
||||
modelPath: "handdetect.json"
|
||||
|
@ -7716,30 +7716,23 @@ var HandDetector = class {
|
|||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const batched = this.model.predict(input);
|
||||
const predictions = tf10.squeeze(batched);
|
||||
tf10.dispose(batched);
|
||||
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await scoresT.data();
|
||||
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
|
||||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
tf10.dispose(rawBoxes);
|
||||
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const filtered = await filteredT.array();
|
||||
tf10.dispose(scoresT);
|
||||
tf10.dispose(filteredT);
|
||||
const t = {};
|
||||
t.batched = this.model.predict(input);
|
||||
t.predictions = tf10.squeeze(t.batched);
|
||||
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of filtered) {
|
||||
if (scores[index] >= config3.hand.minConfidence) {
|
||||
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
|
||||
tf10.dispose(rawPalmLandmarks);
|
||||
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
|
||||
for (const index of nms) {
|
||||
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
}
|
||||
tf10.dispose(predictions);
|
||||
tf10.dispose(boxes);
|
||||
for (const tensor2 of Object.keys(t))
|
||||
tf10.dispose(t[tensor2]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
|
@ -7927,7 +7920,7 @@ var HandPipeline = class {
|
|||
tf11.dispose(handImage);
|
||||
const confidence = (await confidenceT.data())[0];
|
||||
tf11.dispose(confidenceT);
|
||||
if (confidence >= config3.hand.minConfidence) {
|
||||
if (confidence >= config3.hand.minConfidence / 4) {
|
||||
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
|
||||
const rawCoords = await keypointsReshaped.array();
|
||||
tf11.dispose(keypoints3);
|
||||
|
@ -8181,6 +8174,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
|
|||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
if (!landmarks)
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
|
@ -8197,8 +8194,6 @@ function estimate(landmarks) {
|
|||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
|
@ -10793,6 +10788,8 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
if (localOptions.drawLabels) {
|
||||
const addHandLabel = (part, title) => {
|
||||
if (!part)
|
||||
return;
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
|
@ -10992,7 +10989,7 @@ function calc(newResult) {
|
|||
for (let i = 0; i < newResult.hand.length; i++) {
|
||||
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
|
||||
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
|
||||
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
|
||||
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
|
||||
const keys = Object.keys(newResult.hand[i].annotations);
|
||||
const annotations3 = {};
|
||||
for (const key of keys) {
|
||||
|
|
|
@ -194,9 +194,9 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 18,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
minConfidence: 0.8,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: 1,
|
||||
landmarks: true,
|
||||
detector: {
|
||||
modelPath: "handdetect.json"
|
||||
|
@ -7715,30 +7715,23 @@ var HandDetector = class {
|
|||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const batched = this.model.predict(input);
|
||||
const predictions = tf10.squeeze(batched);
|
||||
tf10.dispose(batched);
|
||||
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await scoresT.data();
|
||||
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
|
||||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
tf10.dispose(rawBoxes);
|
||||
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const filtered = await filteredT.array();
|
||||
tf10.dispose(scoresT);
|
||||
tf10.dispose(filteredT);
|
||||
const t = {};
|
||||
t.batched = this.model.predict(input);
|
||||
t.predictions = tf10.squeeze(t.batched);
|
||||
t.scores = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf10.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
t.nms = await tf10.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
|
||||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of filtered) {
|
||||
if (scores[index] >= config3.hand.minConfidence) {
|
||||
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
|
||||
tf10.dispose(rawPalmLandmarks);
|
||||
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
|
||||
for (const index of nms) {
|
||||
const palmBox = tf10.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(tf10.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
}
|
||||
tf10.dispose(predictions);
|
||||
tf10.dispose(boxes);
|
||||
for (const tensor2 of Object.keys(t))
|
||||
tf10.dispose(t[tensor2]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
|
@ -7926,7 +7919,7 @@ var HandPipeline = class {
|
|||
tf11.dispose(handImage);
|
||||
const confidence = (await confidenceT.data())[0];
|
||||
tf11.dispose(confidenceT);
|
||||
if (confidence >= config3.hand.minConfidence) {
|
||||
if (confidence >= config3.hand.minConfidence / 4) {
|
||||
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
|
||||
const rawCoords = await keypointsReshaped.array();
|
||||
tf11.dispose(keypoints3);
|
||||
|
@ -8180,6 +8173,10 @@ function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes)
|
|||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
if (!landmarks)
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
|
@ -8196,8 +8193,6 @@ function estimate(landmarks) {
|
|||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
|
@ -10792,6 +10787,8 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
if (localOptions.drawLabels) {
|
||||
const addHandLabel = (part, title) => {
|
||||
if (!part)
|
||||
return;
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + 2 * part[part.length - 1][2]}, ${127.5 - 2 * part[part.length - 1][2]}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
|
@ -10991,7 +10988,7 @@ function calc(newResult) {
|
|||
for (let i = 0; i < newResult.hand.length; i++) {
|
||||
const box6 = newResult.hand[i].box.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor);
|
||||
const boxRaw3 = newResult.hand[i].boxRaw.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor);
|
||||
const keypoints3 = newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor));
|
||||
const keypoints3 = newResult.hand[i].keypoints ? newResult.hand[i].keypoints.map((landmark, j) => landmark.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) : [];
|
||||
const keys = Object.keys(newResult.hand[i].annotations);
|
||||
const annotations3 = {};
|
||||
for (const key of keys) {
|
||||
|
|
|
@ -2173,9 +2173,9 @@ var require_seedrandom4 = __commonJS({
|
|||
}
|
||||
});
|
||||
|
||||
// (disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js
|
||||
// (disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js
|
||||
var require_string_decoder = __commonJS({
|
||||
"(disabled):node_modules/.pnpm/string_decoder@1.1.1/node_modules/string_decoder/lib/string_decoder.js"() {
|
||||
"(disabled):node_modules/.pnpm/string_decoder@1.3.0/node_modules/string_decoder/lib/string_decoder.js"() {
|
||||
}
|
||||
});
|
||||
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -66,7 +66,7 @@
|
|||
"@tensorflow/tfjs-layers": "^3.9.0",
|
||||
"@tensorflow/tfjs-node": "^3.9.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.9.0",
|
||||
"@types/node": "^16.7.8",
|
||||
"@types/node": "^16.7.10",
|
||||
"@typescript-eslint/eslint-plugin": "^4.30.0",
|
||||
"@typescript-eslint/parser": "^4.30.0",
|
||||
"@vladmandic/pilogger": "^0.2.18",
|
||||
|
|
|
@ -1,22 +1,22 @@
|
|||
2021-08-31 18:22:47 [36mINFO: [39m @vladmandic/human version 2.1.5
|
||||
2021-08-31 18:22:47 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-08-31 18:22:47 [36mINFO: [39m Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"}
|
||||
2021-08-31 18:22:47 [36mINFO: [39m Clean: ["dist/*","types/*","typedoc/*"]
|
||||
2021-08-31 18:22:47 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: node type: node: {"imports":47,"importBytes":456300,"outputBytes":396489,"outputFiles":"dist/human.node.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: nodeGPU type: node: {"imports":47,"importBytes":456308,"outputBytes":396493,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: nodeWASM type: node: {"imports":47,"importBytes":456375,"outputBytes":396565,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-31 18:22:47 [35mSTATE:[39m target: browserNoBundle type: esm: {"imports":47,"importBytes":456239,"outputBytes":255270,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-08-31 18:22:48 [35mSTATE:[39m target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-31 18:22:48 [35mSTATE:[39m target: browserBundle type: iife: {"imports":47,"importBytes":2798929,"outputBytes":1391500,"outputFiles":"dist/human.js"}
|
||||
2021-08-31 18:22:49 [35mSTATE:[39m target: browserBundle type: esm: {"imports":47,"importBytes":2798929,"outputBytes":1391492,"outputFiles":"dist/human.esm.js"}
|
||||
2021-08-31 18:22:49 [36mINFO: [39m Running Linter: ["server/","src/","tfjs/","test/","demo/"]
|
||||
2021-08-31 18:23:12 [36mINFO: [39m Linter complete: files: 84 errors: 0 warnings: 0
|
||||
2021-08-31 18:23:12 [36mINFO: [39m Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-08-31 18:23:12 [36mINFO: [39m Generate Typings: ["src/human.ts"] outDir: ["types"]
|
||||
2021-08-31 18:23:26 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
|
||||
2021-08-31 18:23:40 [36mINFO: [39m Documentation generated at /home/vlado/dev/human/typedoc 1
|
||||
2021-09-02 08:49:02 [36mINFO: [39m @vladmandic/human version 2.1.5
|
||||
2021-09-02 08:49:02 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-09-02 08:49:02 [36mINFO: [39m Toolchain: {"tfjs":"3.9.0","esbuild":"0.12.24","typescript":"4.4.2","typedoc":"0.21.9","eslint":"7.32.0"}
|
||||
2021-09-02 08:49:02 [36mINFO: [39m Clean: ["dist/*","types/*","typedoc/*"]
|
||||
2021-09-02 08:49:02 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: node type: node: {"imports":47,"importBytes":456773,"outputBytes":396371,"outputFiles":"dist/human.node.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: nodeGPU type: node: {"imports":47,"importBytes":456781,"outputBytes":396375,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: nodeWASM type: node: {"imports":47,"importBytes":456848,"outputBytes":396447,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-09-02 08:49:02 [35mSTATE:[39m target: browserNoBundle type: esm: {"imports":47,"importBytes":456712,"outputBytes":255375,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-09-02 08:49:03 [35mSTATE:[39m target: browserBundle type: tfjs: {"modules":1174,"moduleBytes":8150347,"imports":7,"importBytes":2168,"outputBytes":2343932,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-09-02 08:49:03 [35mSTATE:[39m target: browserBundle type: iife: {"imports":47,"importBytes":2799402,"outputBytes":1391653,"outputFiles":"dist/human.js"}
|
||||
2021-09-02 08:49:04 [35mSTATE:[39m target: browserBundle type: esm: {"imports":47,"importBytes":2799402,"outputBytes":1391645,"outputFiles":"dist/human.esm.js"}
|
||||
2021-09-02 08:49:04 [36mINFO: [39m Running Linter: ["server/","src/","tfjs/","test/","demo/"]
|
||||
2021-09-02 08:49:28 [36mINFO: [39m Linter complete: files: 84 errors: 0 warnings: 0
|
||||
2021-09-02 08:49:28 [36mINFO: [39m Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-09-02 08:49:28 [36mINFO: [39m Generate Typings: ["src/human.ts"] outDir: ["types"]
|
||||
2021-09-02 08:49:42 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
|
||||
2021-09-02 08:49:56 [36mINFO: [39m Documentation generated at /home/vlado/dev/human/typedoc 1
|
||||
|
|
|
@ -331,9 +331,9 @@ const config: Config = {
|
|||
// e.g., if model is running st 25 FPS, we can re-use existing bounding
|
||||
// box for updated hand skeleton analysis as the hand probably
|
||||
// hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
minConfidence: 0.1, // threshold for discarding a prediction
|
||||
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
|
||||
maxDetected: 2, // maximum number of hands detected in the input
|
||||
minConfidence: 0.8, // threshold for discarding a prediction
|
||||
iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed
|
||||
maxDetected: 1, // maximum number of hands detected in the input
|
||||
// should be set to the minimum number for performance
|
||||
landmarks: true, // detect hand landmarks or just hand boundary box
|
||||
detector: {
|
||||
|
|
|
@ -407,6 +407,7 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
}
|
||||
if (localOptions.drawLabels) {
|
||||
const addHandLabel = (part, title) => {
|
||||
if (!part) return;
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[part.length - 1][2])}, ${127.5 - (2 * part[part.length - 1][2])}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
|
|
|
@ -167,6 +167,11 @@ export function estimate(landmarks) {
|
|||
// step 1: calculate slopes
|
||||
const slopesXY: Array<number[]> = [];
|
||||
const slopesYZ: Array<number[]> = [];
|
||||
const fingerCurls: Array<number> = [];
|
||||
const fingerDirections: Array<number> = [];
|
||||
if (!landmarks) return { curls: fingerCurls, directions: fingerDirections };
|
||||
|
||||
// step 1: calculate slopes
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY: Array<number> = [];
|
||||
|
@ -186,8 +191,6 @@ export function estimate(landmarks) {
|
|||
}
|
||||
|
||||
// step 2: calculate orientations
|
||||
const fingerCurls: Array<number> = [];
|
||||
const fingerDirections: Array<number> = [];
|
||||
for (const finger of Finger.all) {
|
||||
// start finger predictions from palm - except for thumb
|
||||
const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;
|
||||
|
|
|
@ -40,31 +40,23 @@ export class HandDetector {
|
|||
}
|
||||
|
||||
async getBoxes(input, config) {
|
||||
const batched = this.model.predict(input) as Tensor;
|
||||
const predictions = tf.squeeze(batched);
|
||||
tf.dispose(batched);
|
||||
const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await scoresT.data();
|
||||
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
|
||||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
tf.dispose(rawBoxes);
|
||||
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
|
||||
const filtered = await filteredT.array();
|
||||
|
||||
tf.dispose(scoresT);
|
||||
tf.dispose(filteredT);
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.batched = this.model.predict(input) as Tensor;
|
||||
t.predictions = tf.squeeze(t.batched);
|
||||
t.scores = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
|
||||
const nms = await t.nms.array() as Array<number>;
|
||||
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
|
||||
for (const index of filtered) {
|
||||
if (scores[index] >= config.hand.minConfidence) {
|
||||
const matchingBox = tf.slice(boxes, [index, 0], [1, -1]);
|
||||
const rawPalmLandmarks = tf.slice(predictions, [index, 5], [1, 14]);
|
||||
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
|
||||
tf.dispose(rawPalmLandmarks);
|
||||
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
|
||||
for (const index of nms) {
|
||||
const palmBox = tf.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(tf.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
// console.log('handdetector:getBoxes', nms.length, index, scores[index], config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence, palmBox.dataSync());
|
||||
}
|
||||
}
|
||||
tf.dispose(predictions);
|
||||
tf.dispose(boxes);
|
||||
for (const tensor of Object.keys(t)) tf.dispose(t[tensor]); // dispose all
|
||||
return hands;
|
||||
}
|
||||
|
||||
|
|
|
@ -85,7 +85,7 @@ export class HandPipeline {
|
|||
// run new detector every skipFrames unless we only want box to start with
|
||||
let boxes;
|
||||
|
||||
// console.log(this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame);
|
||||
// console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
|
||||
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image, config);
|
||||
this.skipped = 0;
|
||||
|
@ -120,7 +120,7 @@ export class HandPipeline {
|
|||
tf.dispose(handImage);
|
||||
const confidence = (await confidenceT.data())[0];
|
||||
tf.dispose(confidenceT);
|
||||
if (confidence >= config.hand.minConfidence) {
|
||||
if (confidence >= config.hand.minConfidence / 4) {
|
||||
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
|
||||
const rawCoords = await keypointsReshaped.array();
|
||||
tf.dispose(keypoints);
|
||||
|
@ -135,6 +135,7 @@ export class HandPipeline {
|
|||
};
|
||||
hands.push(result);
|
||||
} else {
|
||||
// console.log('handpipeline:estimateHands low', confidence);
|
||||
this.storedBoxes[i] = null;
|
||||
}
|
||||
tf.dispose(keypoints);
|
||||
|
|
|
@ -148,8 +148,8 @@ export class Human {
|
|||
* @param userConfig: {@link Config}
|
||||
*/
|
||||
constructor(userConfig?: Config | Record<string, unknown>) {
|
||||
Human.version = app.version;
|
||||
Object.defineProperty(this, 'version', { value: app.version });
|
||||
Human.version = app.version; // expose version property on instance of class
|
||||
Object.defineProperty(this, 'version', { value: app.version }); // expose version property directly on class itself
|
||||
defaults.wasmPath = `https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tf.version_core}/dist/`;
|
||||
this.config = mergeDeep(defaults, userConfig || {});
|
||||
this.tf = tf;
|
||||
|
@ -427,6 +427,7 @@ export class Human {
|
|||
const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff);
|
||||
// if difference is above 10x threshold, don't use last value to force reset cache for significant change of scenes or images
|
||||
this.#lastCacheDiff = diff > 10 * this.config.cacheSensitivity ? 0 : diff;
|
||||
// console.log('skipFrame', skipFrame, this.config.cacheSensitivity, diff);
|
||||
return skipFrame;
|
||||
}
|
||||
|
||||
|
|
|
@ -59,9 +59,10 @@ export function calc(newResult: Result): Result {
|
|||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
const keypoints = newResult.hand[i].keypoints // update landmarks
|
||||
const keypoints = newResult.hand[i].keypoints ? newResult.hand[i].keypoints // update landmarks
|
||||
.map((landmark, j) => landmark
|
||||
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number]);
|
||||
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number])
|
||||
: [];
|
||||
const keys = Object.keys(newResult.hand[i].annotations); // update annotations
|
||||
const annotations = {};
|
||||
for (const key of keys) {
|
||||
|
|
Loading…
Reference in New Issue