performance and memory optimizations

pull/233/head
Vladimir Mandic 2021-11-05 11:28:06 -04:00
parent a5865c8164
commit 73f02855aa
27 changed files with 4446 additions and 1283 deletions

View File

@ -53,3 +53,5 @@ Object detection using CenterNet or NanoDet models is not working when using WAS
<https://github.com/tensorflow/tfjs/issues/4824>
<br><hr><br>
> const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); // modulus op implemented in tf

View File

@ -327,10 +327,10 @@ async function drawResults(input) {
`;
ui.framesDraw++;
ui.lastFrame = human.now();
// if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded
if (ui.buffered) {
if (isLive(input)) {
ui.drawThread = requestAnimationFrame(() => drawResults(input));
// ui.drawThread = requestAnimationFrame(() => drawResults(input));
ui.drawThread = setTimeout(() => drawResults(input), 25);
} else {
cancelAnimationFrame(ui.drawThread);
videoPause();

View File

@ -23,7 +23,7 @@
<body>
<canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
<video id="video" playsinline style="display: none"></video>
<pre id="status" style="position: absolute; top: 20px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
<pre id="status" style="position: absolute; top: 12px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
<pre id="log" style="padding: 8px"></pre>
<div id="performance" style="position: absolute; bottom: 0; width: 100%; padding: 8px; font-size: 0.8rem;"></div>
</body>

View File

@ -6,14 +6,13 @@
// demo/typescript/index.ts
import Human from "../../dist/human.esm.js";
var config = {
modelBasePath: "../../models",
backend: "humangl",
async: true
var humanConfig = {
modelBasePath: "../../models"
};
var human = new Human(config);
human.env.perfadd = false;
var result;
var human = new Human(humanConfig);
human.env["perfadd"] = false;
human.draw.options.font = 'small-caps 24px "Lato"';
human.draw.options.lineHeight = 24;
var dom = {
video: document.getElementById("video"),
canvas: document.getElementById("canvas"),
@ -21,20 +20,17 @@ var dom = {
fps: document.getElementById("status"),
perf: document.getElementById("performance")
};
var timestamp = { detect: 0, draw: 0, tensors: 0 };
var fps = { detect: 0, draw: 0 };
var log = (...msg) => {
dom.log.innerText += msg.join(" ") + "\n";
console.log(...msg);
};
var status = (msg) => {
dom.fps.innerText = msg;
};
var perf = (msg) => {
dom.perf.innerText = "performance: " + JSON.stringify(msg).replace(/"|{|}/g, "").replace(/,/g, " | ");
};
var status = (msg) => dom.fps.innerText = msg;
var perf = (msg) => dom.perf.innerText = "tensors:" + human.tf.memory().numTensors + " | performance: " + JSON.stringify(msg).replace(/"|{|}/g, "").replace(/,/g, " | ");
async function webCam() {
status("starting webcam...");
const options = { audio: false, video: { facingMode: "user", resizeMode: "crop-and-scale", width: { ideal: document.body.clientWidth } } };
const options = { audio: false, video: { facingMode: "user", resizeMode: "none", width: { ideal: document.body.clientWidth } } };
const stream = await navigator.mediaDevices.getUserMedia(options);
const ready = new Promise((resolve) => {
dom.video.onloadeddata = () => resolve(true);
@ -57,34 +53,39 @@ async function webCam() {
};
}
async function detectionLoop() {
const t0 = human.now();
if (!dom.video.paused) {
result = await human.detect(dom.video);
await human.detect(dom.video);
const tensors = human.tf.memory().numTensors;
if (tensors - timestamp.tensors !== 0)
log("allocated tensors:", tensors - timestamp.tensors);
timestamp.tensors = tensors;
}
const t1 = human.now();
fps.detect = 1e3 / (t1 - t0);
const now = human.now();
fps.detect = 1e3 / (now - timestamp.detect);
timestamp.detect = now;
requestAnimationFrame(detectionLoop);
}
async function drawLoop() {
const t0 = human.now();
if (!dom.video.paused) {
const interpolated = await human.next(result);
const interpolated = await human.next(human.result);
await human.draw.canvas(dom.video, dom.canvas);
await human.draw.all(dom.canvas, interpolated);
perf(interpolated.performance);
}
const t1 = human.now();
fps.draw = 1e3 / (t1 - t0);
status(dom.video.paused ? "paused" : `fps: ${fps.detect.toFixed(1).padStart(5, " ")} detect / ${fps.draw.toFixed(1).padStart(5, " ")} draw`);
requestAnimationFrame(drawLoop);
const now = human.now();
fps.draw = 1e3 / (now - timestamp.draw);
timestamp.draw = now;
status(dom.video.paused ? "paused" : `fps: ${fps.detect.toFixed(1).padStart(5, " ")} detect | ${fps.draw.toFixed(1).padStart(5, " ")} draw`);
setTimeout(drawLoop, 30);
}
async function main() {
log("human version:", human.version, "tfjs:", human.tf.version_core);
log("human version:", human.version, "tfjs version:", human.tf.version_core);
log("platform:", human.env.platform, "agent:", human.env.agent);
status("loading...");
await human.load();
log("backend:", human.tf.getBackend(), "| available:", human.env.backends);
log("loaded models:" + Object.values(human.models).filter((model) => model !== null).length);
status("initializing...");
log("backend:", human.tf.getBackend(), "available:", human.env.backends);
await human.warmup();
await webCam();
await detectionLoop();

File diff suppressed because one or more lines are too long

View File

@ -11,46 +11,45 @@
import Human from '../../dist/human.esm.js'; // equivalent of @vladmandic/human
const config = {
const humanConfig = { // user configuration for human, used to fine-tune behavior
modelBasePath: '../../models',
backend: 'humangl',
async: true,
// face: { enabled: true, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
// backend: 'humangl',
// async: true,
// face: { enabled: false, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
// body: { enabled: false },
// hand: { enabled: false },
// object: { enabled: false },
// gesture: { enabled: true },
};
const human = new Human(config);
human.env.perfadd = false;
let result;
const human = new Human(humanConfig); // create instance of human with overrides from user configuration
const dom = {
human.env['perfadd'] = false; // is performance data showing instant or total values
human.draw.options.font = 'small-caps 24px "Lato"'; // set font used to draw labels when using draw methods
human.draw.options.lineHeight = 24;
const dom = { // grab instances of dom objects so we dont have to look them up later
video: document.getElementById('video') as HTMLVideoElement,
canvas: document.getElementById('canvas') as HTMLCanvasElement,
log: document.getElementById('log') as HTMLPreElement,
fps: document.getElementById('status') as HTMLPreElement,
perf: document.getElementById('performance') as HTMLDivElement,
};
const timestamp = { detect: 0, draw: 0, tensors: 0 }; // holds information used to calculate performance and possible memory leaks
const fps = { detect: 0, draw: 0 }; // holds calculated fps information for both detect and screen refresh
const fps = { detect: 0, draw: 0 };
const log = (...msg) => {
const log = (...msg) => { // helper method to output messages
dom.log.innerText += msg.join(' ') + '\n';
// eslint-disable-next-line no-console
console.log(...msg);
};
const status = (msg) => {
dom.fps.innerText = msg;
};
const perf = (msg) => {
dom.perf.innerText = 'performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | ');
};
const status = (msg) => dom.fps.innerText = msg; // print status element
const perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
async function webCam() {
async function webCam() { // initialize webcam
status('starting webcam...');
const options = { audio: false, video: { facingMode: 'user', resizeMode: 'crop-and-scale', width: { ideal: document.body.clientWidth } } };
// @ts-ignore resizeMode is not yet defined in tslib
const options: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };
const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);
const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });
dom.video.srcObject = stream;
@ -63,47 +62,53 @@ async function webCam() {
const settings: MediaTrackSettings | string = track.getSettings ? track.getSettings() : '';
const constraints: MediaTrackConstraints | string = track.getConstraints ? track.getConstraints() : '';
log('video:', dom.video.videoWidth, dom.video.videoHeight, track.label, { stream, track, settings, constraints, capabilities });
dom.canvas.onclick = () => {
dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click
if (dom.video.paused) dom.video.play();
else dom.video.pause();
};
}
async function detectionLoop() {
const t0 = human.now();
async function detectionLoop() { // main detection loop
if (!dom.video.paused) {
result = await human.detect(dom.video);
// console.log('profiling data:', await human.profile(dom.video));
await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result
const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks
if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak
timestamp.tensors = tensors;
}
const t1 = human.now();
fps.detect = 1000 / (t1 - t0);
requestAnimationFrame(detectionLoop);
const now = human.now();
fps.detect = 1000 / (now - timestamp.detect);
timestamp.detect = now;
requestAnimationFrame(detectionLoop); // start new frame immediately
}
async function drawLoop() {
const t0 = human.now();
async function drawLoop() { // main screen refresh loop
if (!dom.video.paused) {
const interpolated = await human.next(result);
await human.draw.canvas(dom.video, dom.canvas);
await human.draw.all(dom.canvas, interpolated);
perf(interpolated.performance);
const interpolated = await human.next(human.result); // smoothen result using last-known results
await human.draw.canvas(dom.video, dom.canvas); // draw canvas to screen
await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.
perf(interpolated.performance); // write performance data
}
const t1 = human.now();
fps.draw = 1000 / (t1 - t0);
status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect / ${fps.draw.toFixed(1).padStart(5, ' ')} draw`);
requestAnimationFrame(drawLoop);
const now = human.now();
fps.draw = 1000 / (now - timestamp.draw);
timestamp.draw = now;
status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect | ${fps.draw.toFixed(1).padStart(5, ' ')} draw`); // write status
// requestAnimationFrame(drawLoop); // refresh at screen refresh rate
setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps
}
async function main() {
log('human version:', human.version, 'tfjs:', human.tf.version_core);
async function main() { // main entry point
log('human version:', human.version, 'tfjs version:', human.tf.version_core);
log('platform:', human.env.platform, 'agent:', human.env.agent);
status('loading...');
await human.load();
await human.load(); // preload all models
log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
log('loaded models:' + Object.values(human.models).filter((model) => model !== null).length);
status('initializing...');
log('backend:', human.tf.getBackend(), 'available:', human.env.backends);
await human.warmup();
await webCam();
await detectionLoop();
await drawLoop();
await human.warmup(); // warmup function to initialize backend for future faster detection
await webCam(); // start webcam
await detectionLoop(); // start detection loop
await drawLoop(); // start draw loop
}
window.onload = main;

View File

@ -196,7 +196,7 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2e3,
skipTime: 1e3,
minConfidence: 0.5,
iouThreshold: 0.2,
maxDetected: -1,
@ -215,7 +215,7 @@ var config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1e3
skipTime: 2e3
},
segmentation: {
enabled: false,
@ -4620,7 +4620,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
var UV7 = VTX7.map((x) => UV468[x]);
// src/face/facemeshutil.ts
var createBox = (startEndTensor) => ({ startPoint: tfjs_esm_exports.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tfjs_esm_exports.slice(startEndTensor, [0, 2], [-1, 2]) });
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
var getClampedBox = (box4, input) => box4 ? [
@ -4791,63 +4790,71 @@ async function load3(config3) {
return model3;
}
function decodeBounds(boxOutputs) {
const boxStarts = tfjs_esm_exports.slice(boxOutputs, [0, 1], [-1, 2]);
const centers = tfjs_esm_exports.add(boxStarts, anchors);
const boxSizes = tfjs_esm_exports.slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = tfjs_esm_exports.div(boxSizes, inputSize);
const centersNormalized = tfjs_esm_exports.div(centers, inputSize);
const halfBoxSize = tfjs_esm_exports.div(boxSizesNormalized, 2);
const starts = tfjs_esm_exports.sub(centersNormalized, halfBoxSize);
const ends = tfjs_esm_exports.add(centersNormalized, halfBoxSize);
const startNormalized = tfjs_esm_exports.mul(starts, inputSize);
const endNormalized = tfjs_esm_exports.mul(ends, inputSize);
const concatAxis = 1;
return tfjs_esm_exports.concat2d([startNormalized, endNormalized], concatAxis);
const t = {};
t.boxStarts = tfjs_esm_exports.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tfjs_esm_exports.add(t.boxStarts, anchors);
t.boxSizes = tfjs_esm_exports.slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = tfjs_esm_exports.div(t.boxSizes, inputSize);
t.centersNormalized = tfjs_esm_exports.div(t.centers, inputSize);
t.halfBoxSize = tfjs_esm_exports.div(t.boxSizesNormalized, 2);
t.starts = tfjs_esm_exports.sub(t.centersNormalized, t.halfBoxSize);
t.ends = tfjs_esm_exports.add(t.centersNormalized, t.halfBoxSize);
t.startNormalized = tfjs_esm_exports.mul(t.starts, inputSize);
t.endNormalized = tfjs_esm_exports.mul(t.ends, inputSize);
const boxes = tfjs_esm_exports.concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
return boxes;
}
async function getBoxes(inputImage, config3) {
var _a, _b, _c, _d;
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
return { boxes: [] };
const [batch, boxes, scores] = tfjs_esm_exports.tidy(() => {
const resizedImage = tfjs_esm_exports.image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = tfjs_esm_exports.sub(tfjs_esm_exports.div(resizedImage, 127.5), 0.5);
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = tfjs_esm_exports.concat([sorted[0], sorted[2]], 2);
const concat512 = tfjs_esm_exports.concat([sorted[1], sorted[3]], 2);
const concat3 = tfjs_esm_exports.concat([concat512, concat384], 1);
batchOut = tfjs_esm_exports.squeeze(concat3, 0);
} else {
batchOut = tfjs_esm_exports.squeeze(res);
}
const boxesOut = decodeBounds(batchOut);
const logits = tfjs_esm_exports.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tfjs_esm_exports.squeeze(tfjs_esm_exports.sigmoid(logits));
return [batchOut, boxesOut, scoresOut];
});
const nmsTensor = await tfjs_esm_exports.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await nmsTensor.array();
tfjs_esm_exports.dispose(nmsTensor);
const annotatedBoxes = [];
const scoresData = await scores.data();
const t = {};
t.resized = tfjs_esm_exports.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tfjs_esm_exports.div(t.resized, 127.5);
t.normalized = tfjs_esm_exports.sub(t.div, 0.5);
const res = model3 == null ? void 0 : model3.execute(t.normalized);
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = tfjs_esm_exports.concat([sorted[0], sorted[2]], 2);
t.concat512 = tfjs_esm_exports.concat([sorted[1], sorted[3]], 2);
t.concat = tfjs_esm_exports.concat([t.concat512, t.concat384], 1);
t.batch = tfjs_esm_exports.squeeze(t.concat, 0);
} else {
t.batch = tfjs_esm_exports.squeeze(res);
}
tfjs_esm_exports.dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = tfjs_esm_exports.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tfjs_esm_exports.sigmoid(t.logits);
t.scores = tfjs_esm_exports.squeeze(t.sigmoid);
t.nms = await tfjs_esm_exports.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await t.nms.array();
const boxes = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
const boundingBox = tfjs_esm_exports.slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tfjs_esm_exports.tidy(() => tfjs_esm_exports.reshape(tfjs_esm_exports.squeeze(tfjs_esm_exports.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
tfjs_esm_exports.dispose(boundingBox);
const b = {};
b.bbox = tfjs_esm_exports.slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = tfjs_esm_exports.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tfjs_esm_exports.squeeze(b.slice);
b.landmarks = tfjs_esm_exports.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tfjs_esm_exports.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tfjs_esm_exports.slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: await b.startPoint.data(),
endPoint: await b.endPoint.data()
},
landmarks: await b.landmarks.array(),
confidence
});
Object.keys(b).forEach((tensor3) => tfjs_esm_exports.dispose(b[tensor3]));
}
}
tfjs_esm_exports.dispose(batch);
tfjs_esm_exports.dispose(boxes);
tfjs_esm_exports.dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
};
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}
// src/body/blazeposecoords.ts
@ -5403,32 +5410,25 @@ async function predict5(image25, config3, idx, count2) {
var _a2, _b2;
const obj = [];
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
const t = {};
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
const resize = tfjs_esm_exports.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
const [red, green, blue] = tfjs_esm_exports.split(resize, 3, 3);
tfjs_esm_exports.dispose(resize);
const redNorm = tfjs_esm_exports.mul(red, rgb[0]);
const greenNorm = tfjs_esm_exports.mul(green, rgb[1]);
const blueNorm = tfjs_esm_exports.mul(blue, rgb[2]);
tfjs_esm_exports.dispose(red);
tfjs_esm_exports.dispose(green);
tfjs_esm_exports.dispose(blue);
const grayscale = tfjs_esm_exports.addN([redNorm, greenNorm, blueNorm]);
tfjs_esm_exports.dispose(redNorm);
tfjs_esm_exports.dispose(greenNorm);
tfjs_esm_exports.dispose(blueNorm);
const normalize = tfjs_esm_exports.tidy(() => tfjs_esm_exports.mul(tfjs_esm_exports.sub(grayscale, 0.5), 2));
tfjs_esm_exports.dispose(grayscale);
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
t.resize = tfjs_esm_exports.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
[t.red, t.green, t.blue] = tfjs_esm_exports.split(t.resize, 3, 3);
t.redNorm = tfjs_esm_exports.mul(t.red, rgb[0]);
t.greenNorm = tfjs_esm_exports.mul(t.green, rgb[1]);
t.blueNorm = tfjs_esm_exports.mul(t.blue, rgb[2]);
t.grayscale = tfjs_esm_exports.addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = tfjs_esm_exports.sub(t.grayscale, 0.5);
t.grayscaleMul = tfjs_esm_exports.mul(t.grayscaleSub, 2);
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
lastTime5 = now();
const data = await emotionT.data();
tfjs_esm_exports.dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
tfjs_esm_exports.dispose(normalize);
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
}
last2[idx] = obj;
lastCount2 = count2;
@ -5585,14 +5585,13 @@ async function predict6(input, config3) {
boxCache = [];
for (const possible of possibleBoxes.boxes) {
const box4 = {
startPoint: await possible.box.startPoint.data(),
endPoint: await possible.box.endPoint.data(),
landmarks: await possible.landmarks.array(),
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence
};
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => tfjs_esm_exports.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped7 = 0;
} else {
skipped7++;
@ -5707,17 +5706,13 @@ async function load9(config3) {
return model9;
}
function enhance(input) {
const image25 = tfjs_esm_exports.tidy(() => {
const tensor3 = input.image || input.tensor || input;
if (!(tensor3 instanceof Tensor))
return null;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return null;
const crop2 = tfjs_esm_exports.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tfjs_esm_exports.mul(crop2, 255);
return norm;
});
return image25;
const tensor3 = input.image || input.tensor || input;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return tensor3;
const crop2 = tfjs_esm_exports.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tfjs_esm_exports.mul(crop2, 255);
tfjs_esm_exports.dispose(crop2);
return norm;
}
async function predict7(image25, config3, idx, count2) {
var _a, _b, _c, _d;
@ -8844,27 +8839,39 @@ var HandDetector = class {
this.doubleInputSizeTensor = tfjs_esm_exports.tensor1d([this.inputSize * 2, this.inputSize * 2]);
}
normalizeBoxes(boxes) {
return tfjs_esm_exports.tidy(() => {
const boxOffsets = tfjs_esm_exports.slice(boxes, [0, 0], [-1, 2]);
const boxSizes = tfjs_esm_exports.slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = tfjs_esm_exports.add(tfjs_esm_exports.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = tfjs_esm_exports.div(boxSizes, this.doubleInputSizeTensor);
const startPoints = tfjs_esm_exports.mul(tfjs_esm_exports.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = tfjs_esm_exports.mul(tfjs_esm_exports.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return tfjs_esm_exports.concat2d([startPoints, endPoints], 1);
});
const t = {};
t.boxOffsets = tfjs_esm_exports.slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = tfjs_esm_exports.slice(boxes, [0, 2], [-1, 2]);
t.div = tfjs_esm_exports.div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = tfjs_esm_exports.add(t.div, this.anchorsTensor);
t.halfBoxSizes = tfjs_esm_exports.div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = tfjs_esm_exports.sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = tfjs_esm_exports.mul(t.sub, this.inputSizeTensor);
t.add = tfjs_esm_exports.add(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = tfjs_esm_exports.mul(t.add, this.inputSizeTensor);
const res = tfjs_esm_exports.concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tfjs_esm_exports.tidy(() => {
const landmarks = tfjs_esm_exports.add(tfjs_esm_exports.div(tfjs_esm_exports.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tfjs_esm_exports.mul(landmarks, this.inputSizeTensor);
});
}
async getBoxes(input, config3) {
const t = {};
t.batched = this.model.execute(input);
t.reshape = tfjs_esm_exports.reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = tfjs_esm_exports.div(t.reshape, this.inputSizeTensor);
t.landmarks = tfjs_esm_exports.add(t.div, this.anchors[index]);
const res = tfjs_esm_exports.mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
return res;
}
async predict(input, config3) {
const t = {};
t.resize = tfjs_esm_exports.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
t.div = tfjs_esm_exports.div(t.resize, 127.5);
t.image = tfjs_esm_exports.sub(t.div, 1);
t.batched = this.model.execute(t.image);
t.predictions = tfjs_esm_exports.squeeze(t.batched);
t.scores = tfjs_esm_exports.tidy(() => tfjs_esm_exports.squeeze(tfjs_esm_exports.sigmoid(tfjs_esm_exports.slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = tfjs_esm_exports.slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = tfjs_esm_exports.sigmoid(t.slice);
t.scores = tfjs_esm_exports.squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = tfjs_esm_exports.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
@ -8872,32 +8879,21 @@ var HandDetector = class {
const nms = await t.nms.array();
const hands = [];
for (const index of nms) {
const palmBox = tfjs_esm_exports.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tfjs_esm_exports.tidy(() => tfjs_esm_exports.reshape(this.normalizeLandmarks(tfjs_esm_exports.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor3 of Object.keys(t))
tfjs_esm_exports.dispose(t[tensor3]);
return hands;
}
async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image25 = tfjs_esm_exports.tidy(() => tfjs_esm_exports.sub(tfjs_esm_exports.div(tfjs_esm_exports.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image25, config3);
tfjs_esm_exports.dispose(image25);
const hands = [];
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
tfjs_esm_exports.dispose(prediction.box);
tfjs_esm_exports.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p = {};
p.box = tfjs_esm_exports.slice(t.norm, [index, 0], [1, -1]);
p.slice = tfjs_esm_exports.slice(t.predictions, [index, 5], [1, 14]);
p.norm = this.normalizeLandmarks(p.slice, index);
p.palmLandmarks = tfjs_esm_exports.reshape(p.norm, [-1, 2]);
const box4 = await p.box.data();
const startPoint = box4.slice(0, 2);
const endPoint = box4.slice(2, 4);
const palmLandmarks = await p.palmLandmarks.array();
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p).forEach((tensor3) => tfjs_esm_exports.dispose(p[tensor3]));
}
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
return hands;
}
};
@ -8976,7 +8972,7 @@ var HandPipeline = class {
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
if (config3.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image25, config3);
boxes = await this.handDetector.predict(image25, config3);
this.skipped = 0;
}
if (config3.skipAllowed)
@ -9569,7 +9565,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
var inputSize6 = [[0, 0], [0, 0]];
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
var faceIndex = 4;
var boxExpandFact = 1.6;
var boxExpandFact = 1.7;
var maxDetectorResolution = 512;
var detectorExpandFact = 1.4;
var skipped9 = Number.MAX_SAFE_INTEGER;
@ -10912,7 +10908,7 @@ var options2 = {
color: "rgba(173, 216, 230, 0.6)",
labelColor: "rgba(173, 216, 230, 1)",
shadowColor: "black",
font: 'small-caps 14px "Segoe UI"',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 18,
lineWidth: 4,
pointSize: 2,
@ -10961,15 +10957,15 @@ function rect(ctx, x, y, width, height, localOptions) {
}
ctx.stroke();
}
function lines(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function lines(ctx, points, localOptions) {
if (points.length < 2)
return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -10978,8 +10974,8 @@ function lines(ctx, points = [], localOptions) {
ctx.fill();
}
}
function curves(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function curves(ctx, points, localOptions) {
if (points.length < 2)
return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
@ -12885,6 +12881,24 @@ var Human = class {
this.performance.warmup = Math.trunc(t1 - t0);
return res;
}
async profile(input, userConfig) {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
for (const kernel of profile.kernels) {
if (kernels[kernel.name])
kernels[kernel.name] += kernel.kernelTimeMs;
else
kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
kernelArr.sort((a, b) => b.ms - a.ms);
kernelArr.length = 20;
const res = {};
for (const kernel of kernelArr)
res[kernel.name] = kernel.ms;
return res;
}
async detect(input, userConfig) {
this.state = "detect";
return new Promise(async (resolve) => {

File diff suppressed because one or more lines are too long

286
dist/human.esm.js vendored
View File

@ -185,7 +185,7 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2e3,
skipTime: 1e3,
minConfidence: 0.5,
iouThreshold: 0.2,
maxDetected: -1,
@ -204,7 +204,7 @@ var config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1e3
skipTime: 2e3
},
segmentation: {
enabled: false,
@ -74978,7 +74978,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
var UV7 = VTX7.map((x) => UV468[x]);
// src/face/facemeshutil.ts
var createBox = (startEndTensor) => ({ startPoint: slice(startEndTensor, [0, 0], [-1, 2]), endPoint: slice(startEndTensor, [0, 2], [-1, 2]) });
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
var getClampedBox = (box4, input2) => box4 ? [
@ -75149,63 +75148,71 @@ async function load3(config3) {
return model4;
}
function decodeBounds(boxOutputs) {
const boxStarts = slice(boxOutputs, [0, 1], [-1, 2]);
const centers = add2(boxStarts, anchors);
const boxSizes = slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = div(boxSizes, inputSize);
const centersNormalized = div(centers, inputSize);
const halfBoxSize = div(boxSizesNormalized, 2);
const starts = sub(centersNormalized, halfBoxSize);
const ends = add2(centersNormalized, halfBoxSize);
const startNormalized = mul(starts, inputSize);
const endNormalized = mul(ends, inputSize);
const concatAxis = 1;
return concat2d([startNormalized, endNormalized], concatAxis);
const t = {};
t.boxStarts = slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = add2(t.boxStarts, anchors);
t.boxSizes = slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = div(t.boxSizes, inputSize);
t.centersNormalized = div(t.centers, inputSize);
t.halfBoxSize = div(t.boxSizesNormalized, 2);
t.starts = sub(t.centersNormalized, t.halfBoxSize);
t.ends = add2(t.centersNormalized, t.halfBoxSize);
t.startNormalized = mul(t.starts, inputSize);
t.endNormalized = mul(t.ends, inputSize);
const boxes = concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
return boxes;
}
async function getBoxes(inputImage, config3) {
var _a, _b, _c, _d;
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
return { boxes: [] };
const [batch, boxes, scores] = tidy(() => {
const resizedImage = image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = sub(div(resizedImage, 127.5), 0.5);
const res = model4 == null ? void 0 : model4.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = concat([sorted[0], sorted[2]], 2);
const concat512 = concat([sorted[1], sorted[3]], 2);
const concat6 = concat([concat512, concat384], 1);
batchOut = squeeze(concat6, 0);
} else {
batchOut = squeeze(res);
}
const boxesOut = decodeBounds(batchOut);
const logits = slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = squeeze(sigmoid(logits));
return [batchOut, boxesOut, scoresOut];
});
const nmsTensor = await image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await nmsTensor.array();
dispose(nmsTensor);
const annotatedBoxes = [];
const scoresData = await scores.data();
const t = {};
t.resized = image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = div(t.resized, 127.5);
t.normalized = sub(t.div, 0.5);
const res = model4 == null ? void 0 : model4.execute(t.normalized);
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = concat([sorted[0], sorted[2]], 2);
t.concat512 = concat([sorted[1], sorted[3]], 2);
t.concat = concat([t.concat512, t.concat384], 1);
t.batch = squeeze(t.concat, 0);
} else {
t.batch = squeeze(res);
}
dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = sigmoid(t.logits);
t.scores = squeeze(t.sigmoid);
t.nms = await image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await t.nms.array();
const boxes = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
const boundingBox = slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tidy(() => reshape(squeeze(slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
dispose(boundingBox);
const b = {};
b.bbox = slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = squeeze(b.slice);
b.landmarks = reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: await b.startPoint.data(),
endPoint: await b.endPoint.data()
},
landmarks: await b.landmarks.array(),
confidence
});
Object.keys(b).forEach((tensor2) => dispose(b[tensor2]));
}
}
dispose(batch);
dispose(boxes);
dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
};
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}
// src/body/blazeposecoords.ts
@ -75761,32 +75768,25 @@ async function predict5(image7, config3, idx, count3) {
var _a2, _b2;
const obj = [];
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
const t = {};
const inputSize8 = (model7 == null ? void 0 : model7.inputs[0].shape) ? model7.inputs[0].shape[2] : 0;
const resize = image.resizeBilinear(image7, [inputSize8, inputSize8], false);
const [red, green, blue] = split(resize, 3, 3);
dispose(resize);
const redNorm = mul(red, rgb[0]);
const greenNorm = mul(green, rgb[1]);
const blueNorm = mul(blue, rgb[2]);
dispose(red);
dispose(green);
dispose(blue);
const grayscale = addN([redNorm, greenNorm, blueNorm]);
dispose(redNorm);
dispose(greenNorm);
dispose(blueNorm);
const normalize = tidy(() => mul(sub(grayscale, 0.5), 2));
dispose(grayscale);
const emotionT = model7 == null ? void 0 : model7.execute(normalize);
t.resize = image.resizeBilinear(image7, [inputSize8, inputSize8], false);
[t.red, t.green, t.blue] = split(t.resize, 3, 3);
t.redNorm = mul(t.red, rgb[0]);
t.greenNorm = mul(t.green, rgb[1]);
t.blueNorm = mul(t.blue, rgb[2]);
t.grayscale = addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = sub(t.grayscale, 0.5);
t.grayscaleMul = mul(t.grayscaleSub, 2);
t.emotion = model7 == null ? void 0 : model7.execute(t.grayscaleMul);
lastTime5 = now();
const data = await emotionT.data();
dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
dispose(normalize);
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
}
last2[idx] = obj;
lastCount2 = count3;
@ -75943,14 +75943,13 @@ async function predict6(input2, config3) {
boxCache = [];
for (const possible of possibleBoxes.boxes) {
const box4 = {
startPoint: await possible.box.startPoint.data(),
endPoint: await possible.box.endPoint.data(),
landmarks: await possible.landmarks.array(),
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence
};
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped7 = 0;
} else {
skipped7++;
@ -76065,17 +76064,13 @@ async function load9(config3) {
return model10;
}
function enhance(input2) {
const image7 = tidy(() => {
const tensor2 = input2.image || input2.tensor || input2;
if (!(tensor2 instanceof Tensor))
return null;
if (!(model10 == null ? void 0 : model10.inputs[0].shape))
return null;
const crop2 = image.resizeBilinear(tensor2, [model10.inputs[0].shape[2], model10.inputs[0].shape[1]], false);
const norm2 = mul(crop2, 255);
return norm2;
});
return image7;
const tensor2 = input2.image || input2.tensor || input2;
if (!(model10 == null ? void 0 : model10.inputs[0].shape))
return tensor2;
const crop2 = image.resizeBilinear(tensor2, [model10.inputs[0].shape[2], model10.inputs[0].shape[1]], false);
const norm2 = mul(crop2, 255);
dispose(crop2);
return norm2;
}
async function predict7(image7, config3, idx, count3) {
var _a, _b, _c, _d;
@ -79202,27 +79197,39 @@ var HandDetector = class {
this.doubleInputSizeTensor = tensor1d([this.inputSize * 2, this.inputSize * 2]);
}
normalizeBoxes(boxes) {
return tidy(() => {
const boxOffsets = slice(boxes, [0, 0], [-1, 2]);
const boxSizes = slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = add2(div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = div(boxSizes, this.doubleInputSizeTensor);
const startPoints = mul(sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = mul(add2(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return concat2d([startPoints, endPoints], 1);
});
const t = {};
t.boxOffsets = slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = slice(boxes, [0, 2], [-1, 2]);
t.div = div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = add2(t.div, this.anchorsTensor);
t.halfBoxSizes = div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = mul(t.sub, this.inputSizeTensor);
t.add = add2(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = mul(t.add, this.inputSizeTensor);
const res = concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tidy(() => {
const landmarks = add2(div(reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return mul(landmarks, this.inputSizeTensor);
});
}
async getBoxes(input2, config3) {
const t = {};
t.batched = this.model.execute(input2);
t.reshape = reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = div(t.reshape, this.inputSizeTensor);
t.landmarks = add2(t.div, this.anchors[index]);
const res = mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
return res;
}
async predict(input2, config3) {
const t = {};
t.resize = image.resizeBilinear(input2, [this.inputSize, this.inputSize]);
t.div = div(t.resize, 127.5);
t.image = sub(t.div, 1);
t.batched = this.model.execute(t.image);
t.predictions = squeeze(t.batched);
t.scores = tidy(() => squeeze(sigmoid(slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = sigmoid(t.slice);
t.scores = squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
@ -79230,32 +79237,21 @@ var HandDetector = class {
const nms = await t.nms.array();
const hands = [];
for (const index of nms) {
const palmBox = slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tidy(() => reshape(this.normalizeLandmarks(slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor2 of Object.keys(t))
dispose(t[tensor2]);
return hands;
}
async estimateHandBounds(input2, config3) {
const inputHeight = input2.shape[1];
const inputWidth = input2.shape[2];
const image7 = tidy(() => sub(div(image.resizeBilinear(input2, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image7, config3);
dispose(image7);
const hands = [];
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
dispose(prediction.box);
dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p2 = {};
p2.box = slice(t.norm, [index, 0], [1, -1]);
p2.slice = slice(t.predictions, [index, 5], [1, 14]);
p2.norm = this.normalizeLandmarks(p2.slice, index);
p2.palmLandmarks = reshape(p2.norm, [-1, 2]);
const box4 = await p2.box.data();
const startPoint = box4.slice(0, 2);
const endPoint = box4.slice(2, 4);
const palmLandmarks = await p2.palmLandmarks.array();
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = scaleBoxCoordinates2(hand3, [input2.shape[2] / this.inputSize, input2.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p2).forEach((tensor2) => dispose(p2[tensor2]));
}
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
return hands;
}
};
@ -79334,7 +79330,7 @@ var HandPipeline = class {
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
if (config3.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image7, config3);
boxes = await this.handDetector.predict(image7, config3);
this.skipped = 0;
}
if (config3.skipAllowed)
@ -79927,7 +79923,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
var inputSize6 = [[0, 0], [0, 0]];
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
var faceIndex = 4;
var boxExpandFact = 1.6;
var boxExpandFact = 1.7;
var maxDetectorResolution = 512;
var detectorExpandFact = 1.4;
var skipped9 = Number.MAX_SAFE_INTEGER;
@ -81270,7 +81266,7 @@ var options2 = {
color: "rgba(173, 216, 230, 0.6)",
labelColor: "rgba(173, 216, 230, 1)",
shadowColor: "black",
font: 'small-caps 14px "Segoe UI"',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 18,
lineWidth: 4,
pointSize: 2,
@ -81319,15 +81315,15 @@ function rect(ctx, x, y, width, height, localOptions) {
}
ctx.stroke();
}
function lines(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function lines(ctx, points, localOptions) {
if (points.length < 2)
return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -81336,8 +81332,8 @@ function lines(ctx, points = [], localOptions) {
ctx.fill();
}
}
function curves(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function curves(ctx, points, localOptions) {
if (points.length < 2)
return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
@ -83243,6 +83239,24 @@ var Human = class {
this.performance.warmup = Math.trunc(t1 - t0);
return res;
}
async profile(input2, userConfig) {
const profile2 = await this.tf.profile(() => this.detect(input2, userConfig));
const kernels = {};
for (const kernel of profile2.kernels) {
if (kernels[kernel.name])
kernels[kernel.name] += kernel.kernelTimeMs;
else
kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
kernelArr.sort((a, b) => b.ms - a.ms);
kernelArr.length = 20;
const res = {};
for (const kernel of kernelArr)
res[kernel.name] = kernel.ms;
return res;
}
async detect(input2, userConfig) {
this.state = "detect";
return new Promise(async (resolve) => {

File diff suppressed because one or more lines are too long

682
dist/human.js vendored

File diff suppressed because one or more lines are too long

286
dist/human.node-gpu.js vendored
View File

@ -231,7 +231,7 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2e3,
skipTime: 1e3,
minConfidence: 0.5,
iouThreshold: 0.2,
maxDetected: -1,
@ -250,7 +250,7 @@ var config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1e3
skipTime: 2e3
},
segmentation: {
enabled: false,
@ -4645,7 +4645,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
var UV7 = VTX7.map((x) => UV468[x]);
// src/face/facemeshutil.ts
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
var getClampedBox = (box4, input) => box4 ? [
@ -4816,63 +4815,71 @@ async function load3(config3) {
return model3;
}
function decodeBounds(boxOutputs) {
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
const centers = tf6.add(boxStarts, anchors);
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
const centersNormalized = tf6.div(centers, inputSize);
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
const starts = tf6.sub(centersNormalized, halfBoxSize);
const ends = tf6.add(centersNormalized, halfBoxSize);
const startNormalized = tf6.mul(starts, inputSize);
const endNormalized = tf6.mul(ends, inputSize);
const concatAxis = 1;
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
const t = {};
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tf6.add(t.boxStarts, anchors);
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
t.centersNormalized = tf6.div(t.centers, inputSize);
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
t.startNormalized = tf6.mul(t.starts, inputSize);
t.endNormalized = tf6.mul(t.ends, inputSize);
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return boxes;
}
async function getBoxes(inputImage, config3) {
var _a, _b, _c, _d;
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
return { boxes: [] };
const [batch, boxes, scores] = tf6.tidy(() => {
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
const concat3 = tf6.concat([concat512, concat384], 1);
batchOut = tf6.squeeze(concat3, 0);
} else {
batchOut = tf6.squeeze(res);
}
const boxesOut = decodeBounds(batchOut);
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
return [batchOut, boxesOut, scoresOut];
});
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await nmsTensor.array();
tf6.dispose(nmsTensor);
const annotatedBoxes = [];
const scoresData = await scores.data();
const t = {};
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tf6.div(t.resized, 127.5);
t.normalized = tf6.sub(t.div, 0.5);
const res = model3 == null ? void 0 : model3.execute(t.normalized);
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
t.concat = tf6.concat([t.concat512, t.concat384], 1);
t.batch = tf6.squeeze(t.concat, 0);
} else {
t.batch = tf6.squeeze(res);
}
tf6.dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf6.sigmoid(t.logits);
t.scores = tf6.squeeze(t.sigmoid);
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await t.nms.array();
const boxes = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
tf6.dispose(boundingBox);
const b = {};
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tf6.squeeze(b.slice);
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: await b.startPoint.data(),
endPoint: await b.endPoint.data()
},
landmarks: await b.landmarks.array(),
confidence
});
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
}
}
tf6.dispose(batch);
tf6.dispose(boxes);
tf6.dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
};
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}
// src/body/blazepose.ts
@ -5438,32 +5445,25 @@ async function predict5(image25, config3, idx, count2) {
var _a2, _b2;
const obj = [];
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
const t = {};
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
const [red, green, blue] = tf10.split(resize, 3, 3);
tf10.dispose(resize);
const redNorm = tf10.mul(red, rgb[0]);
const greenNorm = tf10.mul(green, rgb[1]);
const blueNorm = tf10.mul(blue, rgb[2]);
tf10.dispose(red);
tf10.dispose(green);
tf10.dispose(blue);
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
tf10.dispose(redNorm);
tf10.dispose(greenNorm);
tf10.dispose(blueNorm);
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
tf10.dispose(grayscale);
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
t.redNorm = tf10.mul(t.red, rgb[0]);
t.greenNorm = tf10.mul(t.green, rgb[1]);
t.blueNorm = tf10.mul(t.blue, rgb[2]);
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
lastTime5 = now();
const data = await emotionT.data();
tf10.dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
tf10.dispose(normalize);
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
}
last2[idx] = obj;
lastCount2 = count2;
@ -5624,14 +5624,13 @@ async function predict6(input, config3) {
boxCache = [];
for (const possible of possibleBoxes.boxes) {
const box4 = {
startPoint: await possible.box.startPoint.data(),
endPoint: await possible.box.endPoint.data(),
landmarks: await possible.landmarks.array(),
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence
};
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped7 = 0;
} else {
skipped7++;
@ -5747,17 +5746,13 @@ async function load9(config3) {
return model9;
}
function enhance(input) {
const image25 = tf13.tidy(() => {
const tensor3 = input.image || input.tensor || input;
if (!(tensor3 instanceof tf13.Tensor))
return null;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return null;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
return norm;
});
return image25;
const tensor3 = input.image || input.tensor || input;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return tensor3;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
tf13.dispose(crop2);
return norm;
}
async function predict7(image25, config3, idx, count2) {
var _a, _b, _c, _d;
@ -8891,27 +8886,39 @@ var HandDetector = class {
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
}
normalizeBoxes(boxes) {
return tf15.tidy(() => {
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return tf15.concat2d([startPoints, endPoints], 1);
});
const t = {};
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tf15.tidy(() => {
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf15.mul(landmarks, this.inputSizeTensor);
});
}
async getBoxes(input, config3) {
const t = {};
t.batched = this.model.execute(input);
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = tf15.div(t.reshape, this.inputSizeTensor);
t.landmarks = tf15.add(t.div, this.anchors[index]);
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
async predict(input, config3) {
const t = {};
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
t.div = tf15.div(t.resize, 127.5);
t.image = tf15.sub(t.div, 1);
t.batched = this.model.execute(t.image);
t.predictions = tf15.squeeze(t.batched);
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = tf15.sigmoid(t.slice);
t.scores = tf15.squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
@ -8919,32 +8926,21 @@ var HandDetector = class {
const nms = await t.nms.array();
const hands = [];
for (const index of nms) {
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor3 of Object.keys(t))
tf15.dispose(t[tensor3]);
return hands;
}
async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image25, config3);
tf15.dispose(image25);
const hands = [];
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
tf15.dispose(prediction.box);
tf15.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p = {};
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
p.norm = this.normalizeLandmarks(p.slice, index);
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
const box4 = await p.box.data();
const startPoint = box4.slice(0, 2);
const endPoint = box4.slice(2, 4);
const palmLandmarks = await p.palmLandmarks.array();
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
}
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return hands;
}
};
@ -9024,7 +9020,7 @@ var HandPipeline = class {
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
if (config3.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image25, config3);
boxes = await this.handDetector.predict(image25, config3);
this.skipped = 0;
}
if (config3.skipAllowed)
@ -9618,7 +9614,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
var inputSize6 = [[0, 0], [0, 0]];
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
var faceIndex = 4;
var boxExpandFact = 1.6;
var boxExpandFact = 1.7;
var maxDetectorResolution = 512;
var detectorExpandFact = 1.4;
var skipped9 = Number.MAX_SAFE_INTEGER;
@ -10971,7 +10967,7 @@ var options2 = {
color: "rgba(173, 216, 230, 0.6)",
labelColor: "rgba(173, 216, 230, 1)",
shadowColor: "black",
font: 'small-caps 14px "Segoe UI"',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 18,
lineWidth: 4,
pointSize: 2,
@ -11020,15 +11016,15 @@ function rect(ctx, x, y, width, height, localOptions) {
}
ctx.stroke();
}
function lines(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function lines(ctx, points, localOptions) {
if (points.length < 2)
return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -11037,8 +11033,8 @@ function lines(ctx, points = [], localOptions) {
ctx.fill();
}
}
function curves(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function curves(ctx, points, localOptions) {
if (points.length < 2)
return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
@ -12948,6 +12944,24 @@ var Human = class {
this.performance.warmup = Math.trunc(t1 - t0);
return res;
}
async profile(input, userConfig) {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
for (const kernel of profile.kernels) {
if (kernels[kernel.name])
kernels[kernel.name] += kernel.kernelTimeMs;
else
kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
kernelArr.sort((a, b) => b.ms - a.ms);
kernelArr.length = 20;
const res = {};
for (const kernel of kernelArr)
res[kernel.name] = kernel.ms;
return res;
}
async detect(input, userConfig) {
this.state = "detect";
return new Promise(async (resolve) => {

View File

@ -232,7 +232,7 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2e3,
skipTime: 1e3,
minConfidence: 0.5,
iouThreshold: 0.2,
maxDetected: -1,
@ -251,7 +251,7 @@ var config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1e3
skipTime: 2e3
},
segmentation: {
enabled: false,
@ -4646,7 +4646,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
var UV7 = VTX7.map((x) => UV468[x]);
// src/face/facemeshutil.ts
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
var getClampedBox = (box4, input) => box4 ? [
@ -4817,63 +4816,71 @@ async function load3(config3) {
return model3;
}
function decodeBounds(boxOutputs) {
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
const centers = tf6.add(boxStarts, anchors);
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
const centersNormalized = tf6.div(centers, inputSize);
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
const starts = tf6.sub(centersNormalized, halfBoxSize);
const ends = tf6.add(centersNormalized, halfBoxSize);
const startNormalized = tf6.mul(starts, inputSize);
const endNormalized = tf6.mul(ends, inputSize);
const concatAxis = 1;
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
const t = {};
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tf6.add(t.boxStarts, anchors);
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
t.centersNormalized = tf6.div(t.centers, inputSize);
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
t.startNormalized = tf6.mul(t.starts, inputSize);
t.endNormalized = tf6.mul(t.ends, inputSize);
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return boxes;
}
async function getBoxes(inputImage, config3) {
var _a, _b, _c, _d;
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
return { boxes: [] };
const [batch, boxes, scores] = tf6.tidy(() => {
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
const concat3 = tf6.concat([concat512, concat384], 1);
batchOut = tf6.squeeze(concat3, 0);
} else {
batchOut = tf6.squeeze(res);
}
const boxesOut = decodeBounds(batchOut);
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
return [batchOut, boxesOut, scoresOut];
});
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await nmsTensor.array();
tf6.dispose(nmsTensor);
const annotatedBoxes = [];
const scoresData = await scores.data();
const t = {};
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tf6.div(t.resized, 127.5);
t.normalized = tf6.sub(t.div, 0.5);
const res = model3 == null ? void 0 : model3.execute(t.normalized);
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
t.concat = tf6.concat([t.concat512, t.concat384], 1);
t.batch = tf6.squeeze(t.concat, 0);
} else {
t.batch = tf6.squeeze(res);
}
tf6.dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf6.sigmoid(t.logits);
t.scores = tf6.squeeze(t.sigmoid);
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await t.nms.array();
const boxes = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
tf6.dispose(boundingBox);
const b = {};
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tf6.squeeze(b.slice);
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: await b.startPoint.data(),
endPoint: await b.endPoint.data()
},
landmarks: await b.landmarks.array(),
confidence
});
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
}
}
tf6.dispose(batch);
tf6.dispose(boxes);
tf6.dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
};
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}
// src/body/blazepose.ts
@ -5439,32 +5446,25 @@ async function predict5(image25, config3, idx, count2) {
var _a2, _b2;
const obj = [];
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
const t = {};
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
const [red, green, blue] = tf10.split(resize, 3, 3);
tf10.dispose(resize);
const redNorm = tf10.mul(red, rgb[0]);
const greenNorm = tf10.mul(green, rgb[1]);
const blueNorm = tf10.mul(blue, rgb[2]);
tf10.dispose(red);
tf10.dispose(green);
tf10.dispose(blue);
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
tf10.dispose(redNorm);
tf10.dispose(greenNorm);
tf10.dispose(blueNorm);
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
tf10.dispose(grayscale);
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
t.redNorm = tf10.mul(t.red, rgb[0]);
t.greenNorm = tf10.mul(t.green, rgb[1]);
t.blueNorm = tf10.mul(t.blue, rgb[2]);
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
lastTime5 = now();
const data = await emotionT.data();
tf10.dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
tf10.dispose(normalize);
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
}
last2[idx] = obj;
lastCount2 = count2;
@ -5625,14 +5625,13 @@ async function predict6(input, config3) {
boxCache = [];
for (const possible of possibleBoxes.boxes) {
const box4 = {
startPoint: await possible.box.startPoint.data(),
endPoint: await possible.box.endPoint.data(),
landmarks: await possible.landmarks.array(),
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence
};
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped7 = 0;
} else {
skipped7++;
@ -5748,17 +5747,13 @@ async function load9(config3) {
return model9;
}
function enhance(input) {
const image25 = tf13.tidy(() => {
const tensor3 = input.image || input.tensor || input;
if (!(tensor3 instanceof tf13.Tensor))
return null;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return null;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
return norm;
});
return image25;
const tensor3 = input.image || input.tensor || input;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return tensor3;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
tf13.dispose(crop2);
return norm;
}
async function predict7(image25, config3, idx, count2) {
var _a, _b, _c, _d;
@ -8892,27 +8887,39 @@ var HandDetector = class {
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
}
normalizeBoxes(boxes) {
return tf15.tidy(() => {
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return tf15.concat2d([startPoints, endPoints], 1);
});
const t = {};
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tf15.tidy(() => {
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf15.mul(landmarks, this.inputSizeTensor);
});
}
async getBoxes(input, config3) {
const t = {};
t.batched = this.model.execute(input);
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = tf15.div(t.reshape, this.inputSizeTensor);
t.landmarks = tf15.add(t.div, this.anchors[index]);
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
async predict(input, config3) {
const t = {};
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
t.div = tf15.div(t.resize, 127.5);
t.image = tf15.sub(t.div, 1);
t.batched = this.model.execute(t.image);
t.predictions = tf15.squeeze(t.batched);
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = tf15.sigmoid(t.slice);
t.scores = tf15.squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
@ -8920,32 +8927,21 @@ var HandDetector = class {
const nms = await t.nms.array();
const hands = [];
for (const index of nms) {
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor3 of Object.keys(t))
tf15.dispose(t[tensor3]);
return hands;
}
async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image25, config3);
tf15.dispose(image25);
const hands = [];
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
tf15.dispose(prediction.box);
tf15.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p = {};
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
p.norm = this.normalizeLandmarks(p.slice, index);
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
const box4 = await p.box.data();
const startPoint = box4.slice(0, 2);
const endPoint = box4.slice(2, 4);
const palmLandmarks = await p.palmLandmarks.array();
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
}
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return hands;
}
};
@ -9025,7 +9021,7 @@ var HandPipeline = class {
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
if (config3.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image25, config3);
boxes = await this.handDetector.predict(image25, config3);
this.skipped = 0;
}
if (config3.skipAllowed)
@ -9619,7 +9615,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
var inputSize6 = [[0, 0], [0, 0]];
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
var faceIndex = 4;
var boxExpandFact = 1.6;
var boxExpandFact = 1.7;
var maxDetectorResolution = 512;
var detectorExpandFact = 1.4;
var skipped9 = Number.MAX_SAFE_INTEGER;
@ -10972,7 +10968,7 @@ var options2 = {
color: "rgba(173, 216, 230, 0.6)",
labelColor: "rgba(173, 216, 230, 1)",
shadowColor: "black",
font: 'small-caps 14px "Segoe UI"',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 18,
lineWidth: 4,
pointSize: 2,
@ -11021,15 +11017,15 @@ function rect(ctx, x, y, width, height, localOptions) {
}
ctx.stroke();
}
function lines(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function lines(ctx, points, localOptions) {
if (points.length < 2)
return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -11038,8 +11034,8 @@ function lines(ctx, points = [], localOptions) {
ctx.fill();
}
}
function curves(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function curves(ctx, points, localOptions) {
if (points.length < 2)
return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
@ -12949,6 +12945,24 @@ var Human = class {
this.performance.warmup = Math.trunc(t1 - t0);
return res;
}
async profile(input, userConfig) {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
for (const kernel of profile.kernels) {
if (kernels[kernel.name])
kernels[kernel.name] += kernel.kernelTimeMs;
else
kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
kernelArr.sort((a, b) => b.ms - a.ms);
kernelArr.length = 20;
const res = {};
for (const kernel of kernelArr)
res[kernel.name] = kernel.ms;
return res;
}
async detect(input, userConfig) {
this.state = "detect";
return new Promise(async (resolve) => {

286
dist/human.node.js vendored
View File

@ -231,7 +231,7 @@ var config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2e3,
skipTime: 1e3,
minConfidence: 0.5,
iouThreshold: 0.2,
maxDetected: -1,
@ -250,7 +250,7 @@ var config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1e3
skipTime: 2e3
},
segmentation: {
enabled: false,
@ -4645,7 +4645,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
var UV7 = VTX7.map((x) => UV468[x]);
// src/face/facemeshutil.ts
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
var getClampedBox = (box4, input) => box4 ? [
@ -4816,63 +4815,71 @@ async function load3(config3) {
return model3;
}
function decodeBounds(boxOutputs) {
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
const centers = tf6.add(boxStarts, anchors);
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
const centersNormalized = tf6.div(centers, inputSize);
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
const starts = tf6.sub(centersNormalized, halfBoxSize);
const ends = tf6.add(centersNormalized, halfBoxSize);
const startNormalized = tf6.mul(starts, inputSize);
const endNormalized = tf6.mul(ends, inputSize);
const concatAxis = 1;
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
const t = {};
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tf6.add(t.boxStarts, anchors);
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
t.centersNormalized = tf6.div(t.centers, inputSize);
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
t.startNormalized = tf6.mul(t.starts, inputSize);
t.endNormalized = tf6.mul(t.ends, inputSize);
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return boxes;
}
async function getBoxes(inputImage, config3) {
var _a, _b, _c, _d;
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
return { boxes: [] };
const [batch, boxes, scores] = tf6.tidy(() => {
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
const concat3 = tf6.concat([concat512, concat384], 1);
batchOut = tf6.squeeze(concat3, 0);
} else {
batchOut = tf6.squeeze(res);
}
const boxesOut = decodeBounds(batchOut);
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
return [batchOut, boxesOut, scoresOut];
});
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await nmsTensor.array();
tf6.dispose(nmsTensor);
const annotatedBoxes = [];
const scoresData = await scores.data();
const t = {};
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tf6.div(t.resized, 127.5);
t.normalized = tf6.sub(t.div, 0.5);
const res = model3 == null ? void 0 : model3.execute(t.normalized);
if (Array.isArray(res)) {
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
t.concat = tf6.concat([t.concat512, t.concat384], 1);
t.batch = tf6.squeeze(t.concat, 0);
} else {
t.batch = tf6.squeeze(res);
}
tf6.dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf6.sigmoid(t.logits);
t.scores = tf6.squeeze(t.sigmoid);
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
const nms = await t.nms.array();
const boxes = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
tf6.dispose(boundingBox);
const b = {};
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tf6.squeeze(b.slice);
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: await b.startPoint.data(),
endPoint: await b.endPoint.data()
},
landmarks: await b.landmarks.array(),
confidence
});
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
}
}
tf6.dispose(batch);
tf6.dispose(boxes);
tf6.dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
};
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}
// src/body/blazepose.ts
@ -5438,32 +5445,25 @@ async function predict5(image25, config3, idx, count2) {
var _a2, _b2;
const obj = [];
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
const t = {};
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
const [red, green, blue] = tf10.split(resize, 3, 3);
tf10.dispose(resize);
const redNorm = tf10.mul(red, rgb[0]);
const greenNorm = tf10.mul(green, rgb[1]);
const blueNorm = tf10.mul(blue, rgb[2]);
tf10.dispose(red);
tf10.dispose(green);
tf10.dispose(blue);
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
tf10.dispose(redNorm);
tf10.dispose(greenNorm);
tf10.dispose(blueNorm);
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
tf10.dispose(grayscale);
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
t.redNorm = tf10.mul(t.red, rgb[0]);
t.greenNorm = tf10.mul(t.green, rgb[1]);
t.blueNorm = tf10.mul(t.blue, rgb[2]);
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
lastTime5 = now();
const data = await emotionT.data();
tf10.dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
tf10.dispose(normalize);
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
}
last2[idx] = obj;
lastCount2 = count2;
@ -5624,14 +5624,13 @@ async function predict6(input, config3) {
boxCache = [];
for (const possible of possibleBoxes.boxes) {
const box4 = {
startPoint: await possible.box.startPoint.data(),
endPoint: await possible.box.endPoint.data(),
landmarks: await possible.landmarks.array(),
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence
};
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped7 = 0;
} else {
skipped7++;
@ -5747,17 +5746,13 @@ async function load9(config3) {
return model9;
}
function enhance(input) {
const image25 = tf13.tidy(() => {
const tensor3 = input.image || input.tensor || input;
if (!(tensor3 instanceof tf13.Tensor))
return null;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return null;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
return norm;
});
return image25;
const tensor3 = input.image || input.tensor || input;
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
return tensor3;
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
const norm = tf13.mul(crop2, 255);
tf13.dispose(crop2);
return norm;
}
async function predict7(image25, config3, idx, count2) {
var _a, _b, _c, _d;
@ -8891,27 +8886,39 @@ var HandDetector = class {
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
}
normalizeBoxes(boxes) {
return tf15.tidy(() => {
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return tf15.concat2d([startPoints, endPoints], 1);
});
const t = {};
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tf15.tidy(() => {
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf15.mul(landmarks, this.inputSizeTensor);
});
}
async getBoxes(input, config3) {
const t = {};
t.batched = this.model.execute(input);
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = tf15.div(t.reshape, this.inputSizeTensor);
t.landmarks = tf15.add(t.div, this.anchors[index]);
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return res;
}
async predict(input, config3) {
const t = {};
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
t.div = tf15.div(t.resize, 127.5);
t.image = tf15.sub(t.div, 1);
t.batched = this.model.execute(t.image);
t.predictions = tf15.squeeze(t.batched);
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = tf15.sigmoid(t.slice);
t.scores = tf15.squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
@ -8919,32 +8926,21 @@ var HandDetector = class {
const nms = await t.nms.array();
const hands = [];
for (const index of nms) {
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor3 of Object.keys(t))
tf15.dispose(t[tensor3]);
return hands;
}
async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image25, config3);
tf15.dispose(image25);
const hands = [];
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
tf15.dispose(prediction.box);
tf15.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p = {};
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
p.norm = this.normalizeLandmarks(p.slice, index);
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
const box4 = await p.box.data();
const startPoint = box4.slice(0, 2);
const endPoint = box4.slice(2, 4);
const palmLandmarks = await p.palmLandmarks.array();
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
}
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
return hands;
}
};
@ -9024,7 +9020,7 @@ var HandPipeline = class {
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
if (config3.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image25, config3);
boxes = await this.handDetector.predict(image25, config3);
this.skipped = 0;
}
if (config3.skipAllowed)
@ -9618,7 +9614,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
var inputSize6 = [[0, 0], [0, 0]];
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
var faceIndex = 4;
var boxExpandFact = 1.6;
var boxExpandFact = 1.7;
var maxDetectorResolution = 512;
var detectorExpandFact = 1.4;
var skipped9 = Number.MAX_SAFE_INTEGER;
@ -10971,7 +10967,7 @@ var options2 = {
color: "rgba(173, 216, 230, 0.6)",
labelColor: "rgba(173, 216, 230, 1)",
shadowColor: "black",
font: 'small-caps 14px "Segoe UI"',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 18,
lineWidth: 4,
pointSize: 2,
@ -11020,15 +11016,15 @@ function rect(ctx, x, y, width, height, localOptions) {
}
ctx.stroke();
}
function lines(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function lines(ctx, points, localOptions) {
if (points.length < 2)
return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -11037,8 +11033,8 @@ function lines(ctx, points = [], localOptions) {
ctx.fill();
}
}
function curves(ctx, points = [], localOptions) {
if (points === void 0 || points.length === 0)
function curves(ctx, points, localOptions) {
if (points.length < 2)
return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
@ -12948,6 +12944,24 @@ var Human = class {
this.performance.warmup = Math.trunc(t1 - t0);
return res;
}
async profile(input, userConfig) {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
for (const kernel of profile.kernels) {
if (kernels[kernel.name])
kernels[kernel.name] += kernel.kernelTimeMs;
else
kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
kernelArr.sort((a, b) => b.ms - a.ms);
kernelArr.length = 20;
const res = {};
for (const kernel of kernelArr)
res[kernel.name] = kernel.ms;
return res;
}
async detect(input, userConfig) {
this.state = "detect";
return new Promise(async (resolve) => {

View File

@ -339,7 +339,7 @@ const config: Config = {
enabled: true,
rotation: true,
skipFrames: 99,
skipTime: 2000,
skipTime: 1000,
minConfidence: 0.50,
iouThreshold: 0.2,
maxDetected: -1,
@ -358,7 +358,7 @@ const config: Config = {
iouThreshold: 0.4,
maxDetected: 10,
skipFrames: 99,
skipTime: 1000,
skipTime: 2000,
},
segmentation: {
enabled: false,

View File

@ -9,6 +9,7 @@ import * as util from './facemeshutil';
import type { Config } from '../config';
import type { Tensor, GraphModel } from '../tfjs/types';
import { env } from '../util/env';
import type { Point } from '../result';
const keypointsCount = 6;
let model: GraphModel | null;
@ -34,63 +35,72 @@ export async function load(config: Config): Promise<GraphModel> {
}
function decodeBounds(boxOutputs) {
const boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
const centers = tf.add(boxStarts, anchors);
const boxSizes = tf.slice(boxOutputs, [0, 3], [-1, 2]);
const boxSizesNormalized = tf.div(boxSizes, inputSize);
const centersNormalized = tf.div(centers, inputSize);
const halfBoxSize = tf.div(boxSizesNormalized, 2);
const starts = tf.sub(centersNormalized, halfBoxSize);
const ends = tf.add(centersNormalized, halfBoxSize);
const startNormalized = tf.mul(starts, inputSize);
const endNormalized = tf.mul(ends, inputSize);
const concatAxis = 1;
return tf.concat2d([startNormalized, endNormalized], concatAxis);
const t: Record<string, Tensor> = {};
t.boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tf.add(t.boxStarts, anchors);
t.boxSizes = tf.slice(boxOutputs, [0, 3], [-1, 2]);
t.boxSizesNormalized = tf.div(t.boxSizes, inputSize);
t.centersNormalized = tf.div(t.centers, inputSize);
t.halfBoxSize = tf.div(t.boxSizesNormalized, 2);
t.starts = tf.sub(t.centersNormalized, t.halfBoxSize);
t.ends = tf.add(t.centersNormalized, t.halfBoxSize);
t.startNormalized = tf.mul(t.starts, inputSize);
t.endNormalized = tf.mul(t.ends, inputSize);
const boxes = tf.concat2d([t.startNormalized, t.endNormalized], 1);
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return boxes;
}
export async function getBoxes(inputImage: Tensor, config: Config) {
// sanity check on input
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return { boxes: [] };
const [batch, boxes, scores] = tf.tidy(() => {
const resizedImage = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
const normalizedImage = tf.sub(tf.div(resizedImage, 127.5), 0.5);
const res = model?.execute(normalizedImage);
let batchOut;
if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
const sorted = res.sort((a, b) => a.size - b.size);
const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
const concat = tf.concat([concat512, concat384], 1);
batchOut = tf.squeeze(concat, 0);
} else {
batchOut = tf.squeeze(res); // when using tfhub model
}
const boxesOut = decodeBounds(batchOut);
const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf.squeeze(tf.sigmoid(logits)); // inside tf.tidy
return [batchOut, boxesOut, scoresOut];
});
const t: Record<string, Tensor> = {};
const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
const nms = await nmsTensor.array();
tf.dispose(nmsTensor);
const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: [number, number] | undefined, confidence: number }> = [];
const scoresData = await scores.data();
t.resized = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tf.div(t.resized, 127.5);
t.normalized = tf.sub(t.div, 0.5);
const res = model?.execute(t.normalized) as Tensor[];
if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
const sorted = res.sort((a, b) => a.size - b.size);
t.concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
t.concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
t.concat = tf.concat([t.concat512, t.concat384], 1);
t.batch = tf.squeeze(t.concat, 0);
} else {
t.batch = tf.squeeze(res); // when using tfhub model
}
tf.dispose(res);
t.boxes = decodeBounds(t.batch);
t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf.sigmoid(t.logits);
t.scores = tf.squeeze(t.sigmoid);
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
const nms = await t.nms.array() as number[];
const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scoresData[nms[i]];
const confidence = scores[nms[i]];
if (confidence > (config.face.detector?.minConfidence || 0)) {
const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
const landmarks = tf.tidy(() => tf.reshape(tf.squeeze(tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: util.createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
tf.dispose(boundingBox);
const b: Record<string, Tensor> = {};
b.bbox = tf.slice(t.boxes, [nms[i], 0], [1, -1]);
b.slice = tf.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tf.squeeze(b.slice);
b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tf.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tf.slice(b.bbox, [0, 2], [-1, 2]);
boxes.push({
box: {
startPoint: (await b.startPoint.data()) as unknown as Point,
endPoint: (await b.endPoint.data()) as unknown as Point,
},
landmarks: (await b.landmarks.array()) as Point[],
confidence,
});
Object.keys(b).forEach((tensor) => tf.dispose(b[tensor]));
}
}
tf.dispose(batch);
tf.dispose(boxes);
tf.dispose(scores);
return {
boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize],
};
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
}

View File

@ -37,14 +37,13 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
boxCache = []; // empty cache
for (const possible of possibleBoxes.boxes) { // extract data from detector
const box: BoxCache = {
startPoint: await possible.box.startPoint.data() as unknown as Point,
endPoint: await possible.box.endPoint.data() as unknown as Point,
landmarks: await possible.landmarks.array() as Array<Point>,
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence,
};
boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
}
possibleBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
skipped = 0;
} else {
skipped++;

View File

@ -37,57 +37,50 @@ export async function load(config: Config): Promise<GraphModel> {
}
export function enhance(input): Tensor {
const image = tf.tidy(() => {
// input received from detector is already normalized to 0..1
// input is also assumed to be straightened
const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf.Tensor)) return null;
// do a tight crop of image and resize it to fit the model
if (!model?.inputs[0].shape) return null; // model has no shape so no point continuing
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
/*
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
const crop = (tensor.shape.length === 3)
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
*/
/*
// just resize to fit the embedding model instead of cropping
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
*/
const tensor = (input.image || input.tensor || input) as Tensor; // input received from detector is already normalized to 0..1, input is also assumed to be straightened
if (!model?.inputs[0].shape) return tensor; // model has no shape so no point continuing
// do a tight crop of image and resize it to fit the model
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
/*
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
const crop = (tensor.shape.length === 3)
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
*/
/*
// just resize to fit the embedding model instead of cropping
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
*/
/*
// convert to black&white to avoid colorization impact
const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const [red, green, blue] = tf.split(crop, 3, 3);
const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4);
*/
/*
// convert to black&white to avoid colorization impact
const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const [red, green, blue] = tf.split(crop, 3, 3);
const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4);
*/
/*
// increase image pseudo-contrast 100%
// (or do it per-channel so mean is done on each channel)
// (or calculate histogram and do it based on histogram)
const mean = merge.mean();
const factor = 2;
const contrast = merge.sub(mean).mul(factor).add(mean);
*/
/*
// increase image pseudo-contrast 100%
// (or do it per-channel so mean is done on each channel)
// (or calculate histogram and do it based on histogram)
const mean = merge.mean();
const factor = 2;
const contrast = merge.sub(mean).mul(factor).add(mean);
*/
/*
// normalize brightness from 0..1
// silly way of creating pseudo-hdr of image
const darken = crop.sub(crop.min());
const lighten = darken.div(darken.max());
*/
const norm = tf.mul(crop, 255);
return norm;
});
return image;
/*
// normalize brightness from 0..1
// silly way of creating pseudo-hdr of image
const darken = crop.sub(crop.min());
const lighten = darken.div(darken.max());
*/
const norm = tf.mul(crop, 255);
tf.dispose(crop);
return norm;
}
export async function predict(image: Tensor, config: Config, idx, count) {

View File

@ -126,7 +126,7 @@ export async function augmentIris(rawCoords, face, config, meshSize) {
tf.dispose(rightEyeCrop);
const eyePredictions = model.execute(combined) as Tensor;
tf.dispose(combined);
const eyePredictionsData = await eyePredictions.data(); // inside tf.tidy
const eyePredictionsData = await eyePredictions.data();
tf.dispose(eyePredictions);
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);

View File

@ -43,35 +43,27 @@ export async function predict(image: Tensor, config: Config, idx, count) {
return new Promise(async (resolve) => {
const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion?.enabled) {
const t: Record<string, Tensor> = {};
const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0;
const resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false);
t.resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false);
// const box = [[0.15, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
// const resize = tf.image.cropAndResize(image, box, [0], [inputSize, inputSize]);
const [red, green, blue] = tf.split(resize, 3, 3);
tf.dispose(resize);
[t.red, t.green, t.blue] = tf.split(t.resize, 3, 3);
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]);
tf.dispose(red);
tf.dispose(green);
tf.dispose(blue);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
tf.dispose(redNorm);
tf.dispose(greenNorm);
tf.dispose(blueNorm);
const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
tf.dispose(grayscale);
const emotionT = model?.execute(normalize) as Tensor; // result is already in range 0..1, no need for additional activation
t.redNorm = tf.mul(t.red, rgb[0]);
t.greenNorm = tf.mul(t.green, rgb[1]);
t.blueNorm = tf.mul(t.blue, rgb[2]);
t.grayscale = tf.addN([t.redNorm, t.greenNorm, t.blueNorm]);
t.grayscaleSub = tf.sub(t.grayscale, 0.5);
t.grayscaleMul = tf.mul(t.grayscaleSub, 2);
t.emotion = model?.execute(t.grayscaleMul) as Tensor; // result is already in range 0..1, no need for additional activation
lastTime = now();
const data = await emotionT.data();
tf.dispose(emotionT);
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
}
obj.sort((a, b) => b.score - a.score);
tf.dispose(normalize);
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
}
last[idx] = obj;
lastCount = count;

View File

@ -7,6 +7,7 @@ import * as tf from '../../dist/tfjs.esm.js';
import * as util from './handposeutil';
import * as anchors from './handposeanchors';
import type { Tensor, GraphModel } from '../tfjs/types';
import type { Point } from '../result';
export class HandDetector {
model: GraphModel;
@ -26,62 +27,64 @@ export class HandDetector {
}
normalizeBoxes(boxes) {
return tf.tidy(() => {
const boxOffsets = tf.slice(boxes, [0, 0], [-1, 2]);
const boxSizes = tf.slice(boxes, [0, 2], [-1, 2]);
const boxCenterPoints = tf.add(tf.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
const halfBoxSizes = tf.div(boxSizes, this.doubleInputSizeTensor);
const startPoints = tf.mul(tf.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
const endPoints = tf.mul(tf.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
return tf.concat2d([startPoints, endPoints], 1);
});
const t: Record<string, Tensor> = {};
t.boxOffsets = tf.slice(boxes, [0, 0], [-1, 2]);
t.boxSizes = tf.slice(boxes, [0, 2], [-1, 2]);
t.div = tf.div(t.boxOffsets, this.inputSizeTensor);
t.boxCenterPoints = tf.add(t.div, this.anchorsTensor);
t.halfBoxSizes = tf.div(t.boxSizes, this.doubleInputSizeTensor);
t.sub = tf.sub(t.boxCenterPoints, t.halfBoxSizes);
t.startPoints = tf.mul(t.sub, this.inputSizeTensor);
t.add = tf.add(t.boxCenterPoints, t.halfBoxSizes);
t.endPoints = tf.mul(t.add, this.inputSizeTensor);
const res = tf.concat2d([t.startPoints, t.endPoints], 1);
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return res;
}
normalizeLandmarks(rawPalmLandmarks, index) {
return tf.tidy(() => {
const landmarks = tf.add(tf.div(tf.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf.mul(landmarks, this.inputSizeTensor);
});
const t: Record<string, Tensor> = {};
t.reshape = tf.reshape(rawPalmLandmarks, [-1, 7, 2]);
t.div = tf.div(t.reshape, this.inputSizeTensor);
t.landmarks = tf.add(t.div, this.anchors[index]);
const res = tf.mul(t.landmarks, this.inputSizeTensor);
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return res;
}
async getBoxes(input, config) {
async predict(input, config): Promise<{ startPoint: Point; endPoint: Point, palmLandmarks: Point[]; confidence: number }[]> {
const t: Record<string, Tensor> = {};
t.batched = this.model.execute(input) as Tensor;
t.resize = tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
t.div = tf.div(t.resize, 127.5);
t.image = tf.sub(t.div, 1);
t.batched = this.model.execute(t.image) as Tensor;
t.predictions = tf.squeeze(t.batched);
t.scores = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(t.predictions, [0, 0], [-1, 1]))));
t.slice = tf.slice(t.predictions, [0, 0], [-1, 1]);
t.sigmoid = tf.sigmoid(t.slice);
t.scores = tf.squeeze(t.sigmoid);
const scores = await t.scores.data();
t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
t.norm = this.normalizeBoxes(t.boxes);
// box detection is flaky so we look for 3x boxes than we need results
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 3 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const nms = await t.nms.array() as Array<number>;
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
const hands: Array<{ startPoint: Point; endPoint: Point; palmLandmarks: Point[]; confidence: number }> = [];
for (const index of nms) {
const palmBox = tf.slice(t.norm, [index, 0], [1, -1]);
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(tf.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
}
for (const tensor of Object.keys(t)) tf.dispose(t[tensor]); // dispose all
return hands;
}
async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image = tf.tidy(() => tf.sub(tf.div(tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image, config);
tf.dispose(image);
const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
if (!predictions || predictions.length === 0) return hands;
for (const prediction of predictions) {
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
tf.dispose(prediction.box);
tf.dispose(prediction.palmLandmarks);
hands.push(util.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
const p: Record<string, Tensor> = {};
p.box = tf.slice(t.norm, [index, 0], [1, -1]);
p.slice = tf.slice(t.predictions, [index, 5], [1, 14]);
p.norm = this.normalizeLandmarks(p.slice, index);
p.palmLandmarks = tf.reshape(p.norm, [-1, 2]);
const box = await p.box.data();
const startPoint = box.slice(0, 2) as unknown as Point;
const endPoint = box.slice(2, 4) as unknown as Point;
const palmLandmarks = await p.palmLandmarks.array();
const hand = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
const scaled = util.scaleBoxCoordinates(hand, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
hands.push(scaled);
Object.keys(p).forEach((tensor) => tf.dispose(p[tensor]));
}
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return hands;
}
}

View File

@ -9,6 +9,7 @@ import type * as detector from './handposedetector';
import type { Tensor, GraphModel } from '../tfjs/types';
import { env } from '../util/env';
import { now } from '../util/util';
import type { Point } from '../result';
const palmBoxEnlargeFactor = 5; // default 3
const handBoxEnlargeFactor = 1.65; // default 1.65
@ -21,7 +22,7 @@ export class HandPipeline {
handDetector: detector.HandDetector;
handPoseModel: GraphModel;
inputSize: number;
storedBoxes: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number } | null>;
storedBoxes: Array<{ startPoint: Point; endPoint: Point; palmLandmarks: Point[]; confidence: number } | null>;
skipped: number;
detectedHands: number;
@ -93,7 +94,7 @@ export class HandPipeline {
const skipTime = (config.hand.skipTime || 0) > (now() - lastTime);
const skipFrame = this.skipped < (config.hand.skipFrames || 0);
if (config.skipAllowed && skipTime && skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image, config);
boxes = await this.handDetector.predict(image, config);
this.skipped = 0;
}
if (config.skipAllowed) this.skipped++;
@ -105,7 +106,7 @@ export class HandPipeline {
// for (const possible of boxes) this.storedBoxes.push(possible);
if (this.storedBoxes.length > 0) useFreshBox = true;
}
const hands: Array<{ landmarks: number[], confidence: number, boxConfidence: number, fingerConfidence: number, box: { topLeft: number[], bottomRight: number[] } }> = [];
const hands: Array<{ landmarks: Point[], confidence: number, boxConfidence: number, fingerConfidence: number, box: { topLeft: Point, bottomRight: Point } }> = [];
// go through working set of boxes
for (let i = 0; i < this.storedBoxes.length; i++) {

View File

@ -1,4 +1,5 @@
import * as tf from '../../dist/tfjs.esm.js';
import type { Point } from '../result';
export function getBoxSize(box) {
return [
@ -27,8 +28,8 @@ export function cutBoxFromImageAndResize(box, image, cropSize) {
}
export function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]] as Point;
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]] as Point;
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
@ -40,8 +41,8 @@ export function enlargeBox(box, factor = 1.5) {
const center = getBoxCenter(box);
const size = getBoxSize(box);
const newHalfSize = [factor * size[0] / 2, factor * size[1] / 2];
const startPoint = [center[0] - newHalfSize[0], center[1] - newHalfSize[1]];
const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]];
const startPoint = [center[0] - newHalfSize[0], center[1] - newHalfSize[1]] as Point;
const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]] as Point;
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
}
@ -50,8 +51,8 @@ export function squarifyBox(box) {
const size = getBoxSize(box);
const maxEdge = Math.max(...size);
const halfSize = maxEdge / 2;
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
const startPoint = [centers[0] - halfSize, centers[1] - halfSize] as Point;
const endPoint = [centers[0] + halfSize, centers[1] + halfSize] as Point;
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
}
@ -61,8 +62,8 @@ export function shiftBox(box, shiftFactor) {
box.endPoint[1] - box.startPoint[1],
];
const shiftVector = [boxSize[0] * shiftFactor[0], boxSize[1] * shiftFactor[1]];
const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]];
const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]];
const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]] as Point;
const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]] as Point;
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
}

View File

@ -24,7 +24,7 @@ const inputSize = [[0, 0], [0, 0]];
const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip'];
const faceIndex = 4;
const boxExpandFact = 1.6;
const boxExpandFact = 1.7;
const maxDetectorResolution = 512;
const detectorExpandFact = 1.4;

View File

@ -347,6 +347,26 @@ export class Human {
return res;
}
/** Run detect with tensorflow profiling
* - result object will contain total exeuction time information for top-20 kernels
* - actual detection object can be accessed via `human.result`
*/
async profile(input: Input, userConfig?: Partial<Config>): Promise<Record<string, number>> {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
for (const kernel of profile.kernels) { // sum kernel time values per kernel
if (kernels[kernel.name]) kernels[kernel.name] += kernel.kernelTimeMs;
else kernels[kernel.name] = kernel.kernelTimeMs;
}
const kernelArr: Array<{ name, ms }> = [];
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] })); // convert to array
kernelArr.sort((a, b) => b.ms - a.ms); // sort
kernelArr.length = 20; // crop
const res: Record<string, number> = {};
for (const kernel of kernelArr) res[kernel.name] = kernel.ms; // create perf objects
return res;
}
/** Main detection method
* - Analyze configuration: {@link Config}
* - Pre-process input: {@link Input}

View File

@ -52,7 +52,7 @@ export const options: DrawOptions = {
color: <string>'rgba(173, 216, 230, 0.6)', // 'lightblue' with light alpha channel
labelColor: <string>'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel
shadowColor: <string>'black',
font: <string>'small-caps 14px "Segoe UI"',
font: <string>'small-caps 16px "Segoe UI"',
lineHeight: <number>18,
lineWidth: <number>4,
pointSize: <number>2,
@ -106,14 +106,14 @@ function rect(ctx: CanvasRenderingContext2D, x, y, width, height, localOptions)
ctx.stroke();
}
function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
if (points === undefined || points.length === 0) return;
function lines(ctx: CanvasRenderingContext2D, points: Point[], localOptions) {
if (points.length < 2) return;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
const z = pt[2] || 0;
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
ctx.lineTo(pt[0], Math.round(pt[1]));
}
ctx.stroke();
@ -123,8 +123,8 @@ function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions
}
}
function curves(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
if (points === undefined || points.length === 0) return;
function curves(ctx: CanvasRenderingContext2D, points: Point[], localOptions) {
if (points.length < 2) return;
if (!localOptions.useCurves || points.length <= 2) {
lines(ctx, points, localOptions);
return;

File diff suppressed because it is too large Load Diff