mirror of https://github.com/vladmandic/human
performance and memory optimizations
parent
a5865c8164
commit
73f02855aa
2
TODO.md
2
TODO.md
|
@ -53,3 +53,5 @@ Object detection using CenterNet or NanoDet models is not working when using WAS
|
|||
<https://github.com/tensorflow/tfjs/issues/4824>
|
||||
|
||||
<br><hr><br>
|
||||
|
||||
> const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); // modulus op implemented in tf
|
||||
|
|
|
@ -327,10 +327,10 @@ async function drawResults(input) {
|
|||
`;
|
||||
ui.framesDraw++;
|
||||
ui.lastFrame = human.now();
|
||||
// if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded
|
||||
if (ui.buffered) {
|
||||
if (isLive(input)) {
|
||||
ui.drawThread = requestAnimationFrame(() => drawResults(input));
|
||||
// ui.drawThread = requestAnimationFrame(() => drawResults(input));
|
||||
ui.drawThread = setTimeout(() => drawResults(input), 25);
|
||||
} else {
|
||||
cancelAnimationFrame(ui.drawThread);
|
||||
videoPause();
|
||||
|
|
|
@ -23,7 +23,7 @@
|
|||
<body>
|
||||
<canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
|
||||
<video id="video" playsinline style="display: none"></video>
|
||||
<pre id="status" style="position: absolute; top: 20px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
|
||||
<pre id="status" style="position: absolute; top: 12px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
|
||||
<pre id="log" style="padding: 8px"></pre>
|
||||
<div id="performance" style="position: absolute; bottom: 0; width: 100%; padding: 8px; font-size: 0.8rem;"></div>
|
||||
</body>
|
||||
|
|
|
@ -6,14 +6,13 @@
|
|||
|
||||
// demo/typescript/index.ts
|
||||
import Human from "../../dist/human.esm.js";
|
||||
var config = {
|
||||
modelBasePath: "../../models",
|
||||
backend: "humangl",
|
||||
async: true
|
||||
var humanConfig = {
|
||||
modelBasePath: "../../models"
|
||||
};
|
||||
var human = new Human(config);
|
||||
human.env.perfadd = false;
|
||||
var result;
|
||||
var human = new Human(humanConfig);
|
||||
human.env["perfadd"] = false;
|
||||
human.draw.options.font = 'small-caps 24px "Lato"';
|
||||
human.draw.options.lineHeight = 24;
|
||||
var dom = {
|
||||
video: document.getElementById("video"),
|
||||
canvas: document.getElementById("canvas"),
|
||||
|
@ -21,20 +20,17 @@ var dom = {
|
|||
fps: document.getElementById("status"),
|
||||
perf: document.getElementById("performance")
|
||||
};
|
||||
var timestamp = { detect: 0, draw: 0, tensors: 0 };
|
||||
var fps = { detect: 0, draw: 0 };
|
||||
var log = (...msg) => {
|
||||
dom.log.innerText += msg.join(" ") + "\n";
|
||||
console.log(...msg);
|
||||
};
|
||||
var status = (msg) => {
|
||||
dom.fps.innerText = msg;
|
||||
};
|
||||
var perf = (msg) => {
|
||||
dom.perf.innerText = "performance: " + JSON.stringify(msg).replace(/"|{|}/g, "").replace(/,/g, " | ");
|
||||
};
|
||||
var status = (msg) => dom.fps.innerText = msg;
|
||||
var perf = (msg) => dom.perf.innerText = "tensors:" + human.tf.memory().numTensors + " | performance: " + JSON.stringify(msg).replace(/"|{|}/g, "").replace(/,/g, " | ");
|
||||
async function webCam() {
|
||||
status("starting webcam...");
|
||||
const options = { audio: false, video: { facingMode: "user", resizeMode: "crop-and-scale", width: { ideal: document.body.clientWidth } } };
|
||||
const options = { audio: false, video: { facingMode: "user", resizeMode: "none", width: { ideal: document.body.clientWidth } } };
|
||||
const stream = await navigator.mediaDevices.getUserMedia(options);
|
||||
const ready = new Promise((resolve) => {
|
||||
dom.video.onloadeddata = () => resolve(true);
|
||||
|
@ -57,34 +53,39 @@ async function webCam() {
|
|||
};
|
||||
}
|
||||
async function detectionLoop() {
|
||||
const t0 = human.now();
|
||||
if (!dom.video.paused) {
|
||||
result = await human.detect(dom.video);
|
||||
await human.detect(dom.video);
|
||||
const tensors = human.tf.memory().numTensors;
|
||||
if (tensors - timestamp.tensors !== 0)
|
||||
log("allocated tensors:", tensors - timestamp.tensors);
|
||||
timestamp.tensors = tensors;
|
||||
}
|
||||
const t1 = human.now();
|
||||
fps.detect = 1e3 / (t1 - t0);
|
||||
const now = human.now();
|
||||
fps.detect = 1e3 / (now - timestamp.detect);
|
||||
timestamp.detect = now;
|
||||
requestAnimationFrame(detectionLoop);
|
||||
}
|
||||
async function drawLoop() {
|
||||
const t0 = human.now();
|
||||
if (!dom.video.paused) {
|
||||
const interpolated = await human.next(result);
|
||||
const interpolated = await human.next(human.result);
|
||||
await human.draw.canvas(dom.video, dom.canvas);
|
||||
await human.draw.all(dom.canvas, interpolated);
|
||||
perf(interpolated.performance);
|
||||
}
|
||||
const t1 = human.now();
|
||||
fps.draw = 1e3 / (t1 - t0);
|
||||
status(dom.video.paused ? "paused" : `fps: ${fps.detect.toFixed(1).padStart(5, " ")} detect / ${fps.draw.toFixed(1).padStart(5, " ")} draw`);
|
||||
requestAnimationFrame(drawLoop);
|
||||
const now = human.now();
|
||||
fps.draw = 1e3 / (now - timestamp.draw);
|
||||
timestamp.draw = now;
|
||||
status(dom.video.paused ? "paused" : `fps: ${fps.detect.toFixed(1).padStart(5, " ")} detect | ${fps.draw.toFixed(1).padStart(5, " ")} draw`);
|
||||
setTimeout(drawLoop, 30);
|
||||
}
|
||||
async function main() {
|
||||
log("human version:", human.version, "tfjs:", human.tf.version_core);
|
||||
log("human version:", human.version, "tfjs version:", human.tf.version_core);
|
||||
log("platform:", human.env.platform, "agent:", human.env.agent);
|
||||
status("loading...");
|
||||
await human.load();
|
||||
log("backend:", human.tf.getBackend(), "| available:", human.env.backends);
|
||||
log("loaded models:" + Object.values(human.models).filter((model) => model !== null).length);
|
||||
status("initializing...");
|
||||
log("backend:", human.tf.getBackend(), "available:", human.env.backends);
|
||||
await human.warmup();
|
||||
await webCam();
|
||||
await detectionLoop();
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -11,46 +11,45 @@
|
|||
|
||||
import Human from '../../dist/human.esm.js'; // equivalent of @vladmandic/human
|
||||
|
||||
const config = {
|
||||
const humanConfig = { // user configuration for human, used to fine-tune behavior
|
||||
modelBasePath: '../../models',
|
||||
backend: 'humangl',
|
||||
async: true,
|
||||
// face: { enabled: true, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
|
||||
// backend: 'humangl',
|
||||
// async: true,
|
||||
// face: { enabled: false, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
|
||||
// body: { enabled: false },
|
||||
// hand: { enabled: false },
|
||||
// object: { enabled: false },
|
||||
// gesture: { enabled: true },
|
||||
};
|
||||
|
||||
const human = new Human(config);
|
||||
human.env.perfadd = false;
|
||||
let result;
|
||||
const human = new Human(humanConfig); // create instance of human with overrides from user configuration
|
||||
|
||||
const dom = {
|
||||
human.env['perfadd'] = false; // is performance data showing instant or total values
|
||||
human.draw.options.font = 'small-caps 24px "Lato"'; // set font used to draw labels when using draw methods
|
||||
human.draw.options.lineHeight = 24;
|
||||
|
||||
const dom = { // grab instances of dom objects so we dont have to look them up later
|
||||
video: document.getElementById('video') as HTMLVideoElement,
|
||||
canvas: document.getElementById('canvas') as HTMLCanvasElement,
|
||||
log: document.getElementById('log') as HTMLPreElement,
|
||||
fps: document.getElementById('status') as HTMLPreElement,
|
||||
perf: document.getElementById('performance') as HTMLDivElement,
|
||||
};
|
||||
const timestamp = { detect: 0, draw: 0, tensors: 0 }; // holds information used to calculate performance and possible memory leaks
|
||||
const fps = { detect: 0, draw: 0 }; // holds calculated fps information for both detect and screen refresh
|
||||
|
||||
const fps = { detect: 0, draw: 0 };
|
||||
|
||||
const log = (...msg) => {
|
||||
const log = (...msg) => { // helper method to output messages
|
||||
dom.log.innerText += msg.join(' ') + '\n';
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(...msg);
|
||||
};
|
||||
const status = (msg) => {
|
||||
dom.fps.innerText = msg;
|
||||
};
|
||||
const perf = (msg) => {
|
||||
dom.perf.innerText = 'performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | ');
|
||||
};
|
||||
const status = (msg) => dom.fps.innerText = msg; // print status element
|
||||
const perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
|
||||
|
||||
async function webCam() {
|
||||
async function webCam() { // initialize webcam
|
||||
status('starting webcam...');
|
||||
const options = { audio: false, video: { facingMode: 'user', resizeMode: 'crop-and-scale', width: { ideal: document.body.clientWidth } } };
|
||||
// @ts-ignore resizeMode is not yet defined in tslib
|
||||
const options: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };
|
||||
const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);
|
||||
const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });
|
||||
dom.video.srcObject = stream;
|
||||
|
@ -63,47 +62,53 @@ async function webCam() {
|
|||
const settings: MediaTrackSettings | string = track.getSettings ? track.getSettings() : '';
|
||||
const constraints: MediaTrackConstraints | string = track.getConstraints ? track.getConstraints() : '';
|
||||
log('video:', dom.video.videoWidth, dom.video.videoHeight, track.label, { stream, track, settings, constraints, capabilities });
|
||||
dom.canvas.onclick = () => {
|
||||
dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click
|
||||
if (dom.video.paused) dom.video.play();
|
||||
else dom.video.pause();
|
||||
};
|
||||
}
|
||||
|
||||
async function detectionLoop() {
|
||||
const t0 = human.now();
|
||||
async function detectionLoop() { // main detection loop
|
||||
if (!dom.video.paused) {
|
||||
result = await human.detect(dom.video);
|
||||
// console.log('profiling data:', await human.profile(dom.video));
|
||||
await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result
|
||||
const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks
|
||||
if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak
|
||||
timestamp.tensors = tensors;
|
||||
}
|
||||
const t1 = human.now();
|
||||
fps.detect = 1000 / (t1 - t0);
|
||||
requestAnimationFrame(detectionLoop);
|
||||
const now = human.now();
|
||||
fps.detect = 1000 / (now - timestamp.detect);
|
||||
timestamp.detect = now;
|
||||
requestAnimationFrame(detectionLoop); // start new frame immediately
|
||||
}
|
||||
|
||||
async function drawLoop() {
|
||||
const t0 = human.now();
|
||||
async function drawLoop() { // main screen refresh loop
|
||||
if (!dom.video.paused) {
|
||||
const interpolated = await human.next(result);
|
||||
await human.draw.canvas(dom.video, dom.canvas);
|
||||
await human.draw.all(dom.canvas, interpolated);
|
||||
perf(interpolated.performance);
|
||||
const interpolated = await human.next(human.result); // smoothen result using last-known results
|
||||
await human.draw.canvas(dom.video, dom.canvas); // draw canvas to screen
|
||||
await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.
|
||||
perf(interpolated.performance); // write performance data
|
||||
}
|
||||
const t1 = human.now();
|
||||
fps.draw = 1000 / (t1 - t0);
|
||||
status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect / ${fps.draw.toFixed(1).padStart(5, ' ')} draw`);
|
||||
requestAnimationFrame(drawLoop);
|
||||
const now = human.now();
|
||||
fps.draw = 1000 / (now - timestamp.draw);
|
||||
timestamp.draw = now;
|
||||
status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect | ${fps.draw.toFixed(1).padStart(5, ' ')} draw`); // write status
|
||||
// requestAnimationFrame(drawLoop); // refresh at screen refresh rate
|
||||
setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log('human version:', human.version, 'tfjs:', human.tf.version_core);
|
||||
async function main() { // main entry point
|
||||
log('human version:', human.version, 'tfjs version:', human.tf.version_core);
|
||||
log('platform:', human.env.platform, 'agent:', human.env.agent);
|
||||
status('loading...');
|
||||
await human.load();
|
||||
await human.load(); // preload all models
|
||||
log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
|
||||
log('loaded models:' + Object.values(human.models).filter((model) => model !== null).length);
|
||||
status('initializing...');
|
||||
log('backend:', human.tf.getBackend(), 'available:', human.env.backends);
|
||||
await human.warmup();
|
||||
await webCam();
|
||||
await detectionLoop();
|
||||
await drawLoop();
|
||||
await human.warmup(); // warmup function to initialize backend for future faster detection
|
||||
await webCam(); // start webcam
|
||||
await detectionLoop(); // start detection loop
|
||||
await drawLoop(); // start draw loop
|
||||
}
|
||||
|
||||
window.onload = main;
|
||||
|
|
|
@ -196,7 +196,7 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2e3,
|
||||
skipTime: 1e3,
|
||||
minConfidence: 0.5,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -215,7 +215,7 @@ var config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1e3
|
||||
skipTime: 2e3
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
@ -4620,7 +4620,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
|
|||
var UV7 = VTX7.map((x) => UV468[x]);
|
||||
|
||||
// src/face/facemeshutil.ts
|
||||
var createBox = (startEndTensor) => ({ startPoint: tfjs_esm_exports.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tfjs_esm_exports.slice(startEndTensor, [0, 2], [-1, 2]) });
|
||||
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
|
||||
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
|
||||
var getClampedBox = (box4, input) => box4 ? [
|
||||
|
@ -4791,63 +4790,71 @@ async function load3(config3) {
|
|||
return model3;
|
||||
}
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = tfjs_esm_exports.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = tfjs_esm_exports.add(boxStarts, anchors);
|
||||
const boxSizes = tfjs_esm_exports.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = tfjs_esm_exports.div(boxSizes, inputSize);
|
||||
const centersNormalized = tfjs_esm_exports.div(centers, inputSize);
|
||||
const halfBoxSize = tfjs_esm_exports.div(boxSizesNormalized, 2);
|
||||
const starts = tfjs_esm_exports.sub(centersNormalized, halfBoxSize);
|
||||
const ends = tfjs_esm_exports.add(centersNormalized, halfBoxSize);
|
||||
const startNormalized = tfjs_esm_exports.mul(starts, inputSize);
|
||||
const endNormalized = tfjs_esm_exports.mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return tfjs_esm_exports.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t = {};
|
||||
t.boxStarts = tfjs_esm_exports.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tfjs_esm_exports.add(t.boxStarts, anchors);
|
||||
t.boxSizes = tfjs_esm_exports.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = tfjs_esm_exports.div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = tfjs_esm_exports.div(t.centers, inputSize);
|
||||
t.halfBoxSize = tfjs_esm_exports.div(t.boxSizesNormalized, 2);
|
||||
t.starts = tfjs_esm_exports.sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = tfjs_esm_exports.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tfjs_esm_exports.mul(t.starts, inputSize);
|
||||
t.endNormalized = tfjs_esm_exports.mul(t.ends, inputSize);
|
||||
const boxes = tfjs_esm_exports.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
return boxes;
|
||||
}
|
||||
async function getBoxes(inputImage, config3) {
|
||||
var _a, _b, _c, _d;
|
||||
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return { boxes: [] };
|
||||
const [batch, boxes, scores] = tfjs_esm_exports.tidy(() => {
|
||||
const resizedImage = tfjs_esm_exports.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = tfjs_esm_exports.sub(tfjs_esm_exports.div(resizedImage, 127.5), 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tfjs_esm_exports.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tfjs_esm_exports.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tfjs_esm_exports.concat([concat512, concat384], 1);
|
||||
batchOut = tfjs_esm_exports.squeeze(concat3, 0);
|
||||
} else {
|
||||
batchOut = tfjs_esm_exports.squeeze(res);
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = tfjs_esm_exports.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tfjs_esm_exports.squeeze(tfjs_esm_exports.sigmoid(logits));
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const nmsTensor = await tfjs_esm_exports.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await nmsTensor.array();
|
||||
tfjs_esm_exports.dispose(nmsTensor);
|
||||
const annotatedBoxes = [];
|
||||
const scoresData = await scores.data();
|
||||
const t = {};
|
||||
t.resized = tfjs_esm_exports.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = tfjs_esm_exports.div(t.resized, 127.5);
|
||||
t.normalized = tfjs_esm_exports.sub(t.div, 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(t.normalized);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = tfjs_esm_exports.concat([sorted[0], sorted[2]], 2);
|
||||
t.concat512 = tfjs_esm_exports.concat([sorted[1], sorted[3]], 2);
|
||||
t.concat = tfjs_esm_exports.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tfjs_esm_exports.squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = tfjs_esm_exports.squeeze(res);
|
||||
}
|
||||
tfjs_esm_exports.dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = tfjs_esm_exports.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tfjs_esm_exports.sigmoid(t.logits);
|
||||
t.scores = tfjs_esm_exports.squeeze(t.sigmoid);
|
||||
t.nms = await tfjs_esm_exports.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await t.nms.array();
|
||||
const boxes = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
|
||||
const boundingBox = tfjs_esm_exports.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tfjs_esm_exports.tidy(() => tfjs_esm_exports.reshape(tfjs_esm_exports.squeeze(tfjs_esm_exports.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
tfjs_esm_exports.dispose(boundingBox);
|
||||
const b = {};
|
||||
b.bbox = tfjs_esm_exports.slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = tfjs_esm_exports.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = tfjs_esm_exports.squeeze(b.slice);
|
||||
b.landmarks = tfjs_esm_exports.reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = tfjs_esm_exports.slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = tfjs_esm_exports.slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: await b.startPoint.data(),
|
||||
endPoint: await b.endPoint.data()
|
||||
},
|
||||
landmarks: await b.landmarks.array(),
|
||||
confidence
|
||||
});
|
||||
Object.keys(b).forEach((tensor3) => tfjs_esm_exports.dispose(b[tensor3]));
|
||||
}
|
||||
}
|
||||
tfjs_esm_exports.dispose(batch);
|
||||
tfjs_esm_exports.dispose(boxes);
|
||||
tfjs_esm_exports.dispose(scores);
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
|
||||
};
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
||||
// src/body/blazeposecoords.ts
|
||||
|
@ -5403,32 +5410,25 @@ async function predict5(image25, config3, idx, count2) {
|
|||
var _a2, _b2;
|
||||
const obj = [];
|
||||
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
|
||||
const t = {};
|
||||
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
|
||||
const resize = tfjs_esm_exports.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
const [red, green, blue] = tfjs_esm_exports.split(resize, 3, 3);
|
||||
tfjs_esm_exports.dispose(resize);
|
||||
const redNorm = tfjs_esm_exports.mul(red, rgb[0]);
|
||||
const greenNorm = tfjs_esm_exports.mul(green, rgb[1]);
|
||||
const blueNorm = tfjs_esm_exports.mul(blue, rgb[2]);
|
||||
tfjs_esm_exports.dispose(red);
|
||||
tfjs_esm_exports.dispose(green);
|
||||
tfjs_esm_exports.dispose(blue);
|
||||
const grayscale = tfjs_esm_exports.addN([redNorm, greenNorm, blueNorm]);
|
||||
tfjs_esm_exports.dispose(redNorm);
|
||||
tfjs_esm_exports.dispose(greenNorm);
|
||||
tfjs_esm_exports.dispose(blueNorm);
|
||||
const normalize = tfjs_esm_exports.tidy(() => tfjs_esm_exports.mul(tfjs_esm_exports.sub(grayscale, 0.5), 2));
|
||||
tfjs_esm_exports.dispose(grayscale);
|
||||
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
|
||||
t.resize = tfjs_esm_exports.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
[t.red, t.green, t.blue] = tfjs_esm_exports.split(t.resize, 3, 3);
|
||||
t.redNorm = tfjs_esm_exports.mul(t.red, rgb[0]);
|
||||
t.greenNorm = tfjs_esm_exports.mul(t.green, rgb[1]);
|
||||
t.blueNorm = tfjs_esm_exports.mul(t.blue, rgb[2]);
|
||||
t.grayscale = tfjs_esm_exports.addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = tfjs_esm_exports.sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = tfjs_esm_exports.mul(t.grayscaleSub, 2);
|
||||
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
|
||||
lastTime5 = now();
|
||||
const data = await emotionT.data();
|
||||
tfjs_esm_exports.dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
|
||||
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tfjs_esm_exports.dispose(normalize);
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
}
|
||||
last2[idx] = obj;
|
||||
lastCount2 = count2;
|
||||
|
@ -5585,14 +5585,13 @@ async function predict6(input, config3) {
|
|||
boxCache = [];
|
||||
for (const possible of possibleBoxes.boxes) {
|
||||
const box4 = {
|
||||
startPoint: await possible.box.startPoint.data(),
|
||||
endPoint: await possible.box.endPoint.data(),
|
||||
landmarks: await possible.landmarks.array(),
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence
|
||||
};
|
||||
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tfjs_esm_exports.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped7 = 0;
|
||||
} else {
|
||||
skipped7++;
|
||||
|
@ -5707,17 +5706,13 @@ async function load9(config3) {
|
|||
return model9;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image25 = tfjs_esm_exports.tidy(() => {
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(tensor3 instanceof Tensor))
|
||||
return null;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return null;
|
||||
const crop2 = tfjs_esm_exports.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tfjs_esm_exports.mul(crop2, 255);
|
||||
return norm;
|
||||
});
|
||||
return image25;
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return tensor3;
|
||||
const crop2 = tfjs_esm_exports.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tfjs_esm_exports.mul(crop2, 255);
|
||||
tfjs_esm_exports.dispose(crop2);
|
||||
return norm;
|
||||
}
|
||||
async function predict7(image25, config3, idx, count2) {
|
||||
var _a, _b, _c, _d;
|
||||
|
@ -8844,27 +8839,39 @@ var HandDetector = class {
|
|||
this.doubleInputSizeTensor = tfjs_esm_exports.tensor1d([this.inputSize * 2, this.inputSize * 2]);
|
||||
}
|
||||
normalizeBoxes(boxes) {
|
||||
return tfjs_esm_exports.tidy(() => {
|
||||
const boxOffsets = tfjs_esm_exports.slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = tfjs_esm_exports.slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = tfjs_esm_exports.add(tfjs_esm_exports.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = tfjs_esm_exports.div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = tfjs_esm_exports.mul(tfjs_esm_exports.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = tfjs_esm_exports.mul(tfjs_esm_exports.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return tfjs_esm_exports.concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t = {};
|
||||
t.boxOffsets = tfjs_esm_exports.slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = tfjs_esm_exports.slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = tfjs_esm_exports.div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = tfjs_esm_exports.add(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = tfjs_esm_exports.div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = tfjs_esm_exports.sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = tfjs_esm_exports.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tfjs_esm_exports.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tfjs_esm_exports.mul(t.add, this.inputSizeTensor);
|
||||
const res = tfjs_esm_exports.concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tfjs_esm_exports.tidy(() => {
|
||||
const landmarks = tfjs_esm_exports.add(tfjs_esm_exports.div(tfjs_esm_exports.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return tfjs_esm_exports.mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const t = {};
|
||||
t.batched = this.model.execute(input);
|
||||
t.reshape = tfjs_esm_exports.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tfjs_esm_exports.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tfjs_esm_exports.add(t.div, this.anchors[index]);
|
||||
const res = tfjs_esm_exports.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
async predict(input, config3) {
|
||||
const t = {};
|
||||
t.resize = tfjs_esm_exports.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tfjs_esm_exports.div(t.resize, 127.5);
|
||||
t.image = tfjs_esm_exports.sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image);
|
||||
t.predictions = tfjs_esm_exports.squeeze(t.batched);
|
||||
t.scores = tfjs_esm_exports.tidy(() => tfjs_esm_exports.squeeze(tfjs_esm_exports.sigmoid(tfjs_esm_exports.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = tfjs_esm_exports.slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tfjs_esm_exports.sigmoid(t.slice);
|
||||
t.scores = tfjs_esm_exports.squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tfjs_esm_exports.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
|
@ -8872,32 +8879,21 @@ var HandDetector = class {
|
|||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = tfjs_esm_exports.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tfjs_esm_exports.tidy(() => tfjs_esm_exports.reshape(this.normalizeLandmarks(tfjs_esm_exports.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor3 of Object.keys(t))
|
||||
tfjs_esm_exports.dispose(t[tensor3]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image25 = tfjs_esm_exports.tidy(() => tfjs_esm_exports.sub(tfjs_esm_exports.div(tfjs_esm_exports.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image25, config3);
|
||||
tfjs_esm_exports.dispose(image25);
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tfjs_esm_exports.dispose(prediction.box);
|
||||
tfjs_esm_exports.dispose(prediction.palmLandmarks);
|
||||
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p = {};
|
||||
p.box = tfjs_esm_exports.slice(t.norm, [index, 0], [1, -1]);
|
||||
p.slice = tfjs_esm_exports.slice(t.predictions, [index, 5], [1, 14]);
|
||||
p.norm = this.normalizeLandmarks(p.slice, index);
|
||||
p.palmLandmarks = tfjs_esm_exports.reshape(p.norm, [-1, 2]);
|
||||
const box4 = await p.box.data();
|
||||
const startPoint = box4.slice(0, 2);
|
||||
const endPoint = box4.slice(2, 4);
|
||||
const palmLandmarks = await p.palmLandmarks.array();
|
||||
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p).forEach((tensor3) => tfjs_esm_exports.dispose(p[tensor3]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor3) => tfjs_esm_exports.dispose(t[tensor3]));
|
||||
return hands;
|
||||
}
|
||||
};
|
||||
|
@ -8976,7 +8972,7 @@ var HandPipeline = class {
|
|||
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
|
||||
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
|
||||
if (config3.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image25, config3);
|
||||
boxes = await this.handDetector.predict(image25, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipAllowed)
|
||||
|
@ -9569,7 +9565,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
|
|||
var inputSize6 = [[0, 0], [0, 0]];
|
||||
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
|
||||
var faceIndex = 4;
|
||||
var boxExpandFact = 1.6;
|
||||
var boxExpandFact = 1.7;
|
||||
var maxDetectorResolution = 512;
|
||||
var detectorExpandFact = 1.4;
|
||||
var skipped9 = Number.MAX_SAFE_INTEGER;
|
||||
|
@ -10912,7 +10908,7 @@ var options2 = {
|
|||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
font: 'small-caps 14px "Segoe UI"',
|
||||
font: 'small-caps 16px "Segoe UI"',
|
||||
lineHeight: 18,
|
||||
lineWidth: 4,
|
||||
pointSize: 2,
|
||||
|
@ -10961,15 +10957,15 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
function lines(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function lines(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -10978,8 +10974,8 @@ function lines(ctx, points = [], localOptions) {
|
|||
ctx.fill();
|
||||
}
|
||||
}
|
||||
function curves(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function curves(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
@ -12885,6 +12881,24 @@ var Human = class {
|
|||
this.performance.warmup = Math.trunc(t1 - t0);
|
||||
return res;
|
||||
}
|
||||
async profile(input, userConfig) {
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile.kernels) {
|
||||
if (kernels[kernel.name])
|
||||
kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else
|
||||
kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
|
||||
kernelArr.sort((a, b) => b.ms - a.ms);
|
||||
kernelArr.length = 20;
|
||||
const res = {};
|
||||
for (const kernel of kernelArr)
|
||||
res[kernel.name] = kernel.ms;
|
||||
return res;
|
||||
}
|
||||
async detect(input, userConfig) {
|
||||
this.state = "detect";
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -185,7 +185,7 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2e3,
|
||||
skipTime: 1e3,
|
||||
minConfidence: 0.5,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -204,7 +204,7 @@ var config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1e3
|
||||
skipTime: 2e3
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
@ -74978,7 +74978,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
|
|||
var UV7 = VTX7.map((x) => UV468[x]);
|
||||
|
||||
// src/face/facemeshutil.ts
|
||||
var createBox = (startEndTensor) => ({ startPoint: slice(startEndTensor, [0, 0], [-1, 2]), endPoint: slice(startEndTensor, [0, 2], [-1, 2]) });
|
||||
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
|
||||
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
|
||||
var getClampedBox = (box4, input2) => box4 ? [
|
||||
|
@ -75149,63 +75148,71 @@ async function load3(config3) {
|
|||
return model4;
|
||||
}
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = add2(boxStarts, anchors);
|
||||
const boxSizes = slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = div(boxSizes, inputSize);
|
||||
const centersNormalized = div(centers, inputSize);
|
||||
const halfBoxSize = div(boxSizesNormalized, 2);
|
||||
const starts = sub(centersNormalized, halfBoxSize);
|
||||
const ends = add2(centersNormalized, halfBoxSize);
|
||||
const startNormalized = mul(starts, inputSize);
|
||||
const endNormalized = mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t = {};
|
||||
t.boxStarts = slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = add2(t.boxStarts, anchors);
|
||||
t.boxSizes = slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = div(t.centers, inputSize);
|
||||
t.halfBoxSize = div(t.boxSizesNormalized, 2);
|
||||
t.starts = sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = add2(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = mul(t.starts, inputSize);
|
||||
t.endNormalized = mul(t.ends, inputSize);
|
||||
const boxes = concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
return boxes;
|
||||
}
|
||||
async function getBoxes(inputImage, config3) {
|
||||
var _a, _b, _c, _d;
|
||||
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return { boxes: [] };
|
||||
const [batch, boxes, scores] = tidy(() => {
|
||||
const resizedImage = image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = sub(div(resizedImage, 127.5), 0.5);
|
||||
const res = model4 == null ? void 0 : model4.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = concat([sorted[1], sorted[3]], 2);
|
||||
const concat6 = concat([concat512, concat384], 1);
|
||||
batchOut = squeeze(concat6, 0);
|
||||
} else {
|
||||
batchOut = squeeze(res);
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = squeeze(sigmoid(logits));
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const nmsTensor = await image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await nmsTensor.array();
|
||||
dispose(nmsTensor);
|
||||
const annotatedBoxes = [];
|
||||
const scoresData = await scores.data();
|
||||
const t = {};
|
||||
t.resized = image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = div(t.resized, 127.5);
|
||||
t.normalized = sub(t.div, 0.5);
|
||||
const res = model4 == null ? void 0 : model4.execute(t.normalized);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = concat([sorted[0], sorted[2]], 2);
|
||||
t.concat512 = concat([sorted[1], sorted[3]], 2);
|
||||
t.concat = concat([t.concat512, t.concat384], 1);
|
||||
t.batch = squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = squeeze(res);
|
||||
}
|
||||
dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = sigmoid(t.logits);
|
||||
t.scores = squeeze(t.sigmoid);
|
||||
t.nms = await image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await t.nms.array();
|
||||
const boxes = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
|
||||
const boundingBox = slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tidy(() => reshape(squeeze(slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
dispose(boundingBox);
|
||||
const b = {};
|
||||
b.bbox = slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = squeeze(b.slice);
|
||||
b.landmarks = reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: await b.startPoint.data(),
|
||||
endPoint: await b.endPoint.data()
|
||||
},
|
||||
landmarks: await b.landmarks.array(),
|
||||
confidence
|
||||
});
|
||||
Object.keys(b).forEach((tensor2) => dispose(b[tensor2]));
|
||||
}
|
||||
}
|
||||
dispose(batch);
|
||||
dispose(boxes);
|
||||
dispose(scores);
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
|
||||
};
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
||||
// src/body/blazeposecoords.ts
|
||||
|
@ -75761,32 +75768,25 @@ async function predict5(image7, config3, idx, count3) {
|
|||
var _a2, _b2;
|
||||
const obj = [];
|
||||
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
|
||||
const t = {};
|
||||
const inputSize8 = (model7 == null ? void 0 : model7.inputs[0].shape) ? model7.inputs[0].shape[2] : 0;
|
||||
const resize = image.resizeBilinear(image7, [inputSize8, inputSize8], false);
|
||||
const [red, green, blue] = split(resize, 3, 3);
|
||||
dispose(resize);
|
||||
const redNorm = mul(red, rgb[0]);
|
||||
const greenNorm = mul(green, rgb[1]);
|
||||
const blueNorm = mul(blue, rgb[2]);
|
||||
dispose(red);
|
||||
dispose(green);
|
||||
dispose(blue);
|
||||
const grayscale = addN([redNorm, greenNorm, blueNorm]);
|
||||
dispose(redNorm);
|
||||
dispose(greenNorm);
|
||||
dispose(blueNorm);
|
||||
const normalize = tidy(() => mul(sub(grayscale, 0.5), 2));
|
||||
dispose(grayscale);
|
||||
const emotionT = model7 == null ? void 0 : model7.execute(normalize);
|
||||
t.resize = image.resizeBilinear(image7, [inputSize8, inputSize8], false);
|
||||
[t.red, t.green, t.blue] = split(t.resize, 3, 3);
|
||||
t.redNorm = mul(t.red, rgb[0]);
|
||||
t.greenNorm = mul(t.green, rgb[1]);
|
||||
t.blueNorm = mul(t.blue, rgb[2]);
|
||||
t.grayscale = addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = mul(t.grayscaleSub, 2);
|
||||
t.emotion = model7 == null ? void 0 : model7.execute(t.grayscaleMul);
|
||||
lastTime5 = now();
|
||||
const data = await emotionT.data();
|
||||
dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
|
||||
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
dispose(normalize);
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
}
|
||||
last2[idx] = obj;
|
||||
lastCount2 = count3;
|
||||
|
@ -75943,14 +75943,13 @@ async function predict6(input2, config3) {
|
|||
boxCache = [];
|
||||
for (const possible of possibleBoxes.boxes) {
|
||||
const box4 = {
|
||||
startPoint: await possible.box.startPoint.data(),
|
||||
endPoint: await possible.box.endPoint.data(),
|
||||
landmarks: await possible.landmarks.array(),
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence
|
||||
};
|
||||
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped7 = 0;
|
||||
} else {
|
||||
skipped7++;
|
||||
|
@ -76065,17 +76064,13 @@ async function load9(config3) {
|
|||
return model10;
|
||||
}
|
||||
function enhance(input2) {
|
||||
const image7 = tidy(() => {
|
||||
const tensor2 = input2.image || input2.tensor || input2;
|
||||
if (!(tensor2 instanceof Tensor))
|
||||
return null;
|
||||
if (!(model10 == null ? void 0 : model10.inputs[0].shape))
|
||||
return null;
|
||||
const crop2 = image.resizeBilinear(tensor2, [model10.inputs[0].shape[2], model10.inputs[0].shape[1]], false);
|
||||
const norm2 = mul(crop2, 255);
|
||||
return norm2;
|
||||
});
|
||||
return image7;
|
||||
const tensor2 = input2.image || input2.tensor || input2;
|
||||
if (!(model10 == null ? void 0 : model10.inputs[0].shape))
|
||||
return tensor2;
|
||||
const crop2 = image.resizeBilinear(tensor2, [model10.inputs[0].shape[2], model10.inputs[0].shape[1]], false);
|
||||
const norm2 = mul(crop2, 255);
|
||||
dispose(crop2);
|
||||
return norm2;
|
||||
}
|
||||
async function predict7(image7, config3, idx, count3) {
|
||||
var _a, _b, _c, _d;
|
||||
|
@ -79202,27 +79197,39 @@ var HandDetector = class {
|
|||
this.doubleInputSizeTensor = tensor1d([this.inputSize * 2, this.inputSize * 2]);
|
||||
}
|
||||
normalizeBoxes(boxes) {
|
||||
return tidy(() => {
|
||||
const boxOffsets = slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = add2(div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = mul(sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = mul(add2(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t = {};
|
||||
t.boxOffsets = slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = add2(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = mul(t.sub, this.inputSizeTensor);
|
||||
t.add = add2(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = mul(t.add, this.inputSizeTensor);
|
||||
const res = concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
return res;
|
||||
}
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tidy(() => {
|
||||
const landmarks = add2(div(reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
}
|
||||
async getBoxes(input2, config3) {
|
||||
const t = {};
|
||||
t.batched = this.model.execute(input2);
|
||||
t.reshape = reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = add2(t.div, this.anchors[index]);
|
||||
const res = mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
return res;
|
||||
}
|
||||
async predict(input2, config3) {
|
||||
const t = {};
|
||||
t.resize = image.resizeBilinear(input2, [this.inputSize, this.inputSize]);
|
||||
t.div = div(t.resize, 127.5);
|
||||
t.image = sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image);
|
||||
t.predictions = squeeze(t.batched);
|
||||
t.scores = tidy(() => squeeze(sigmoid(slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = sigmoid(t.slice);
|
||||
t.scores = squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
|
@ -79230,32 +79237,21 @@ var HandDetector = class {
|
|||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tidy(() => reshape(this.normalizeLandmarks(slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor2 of Object.keys(t))
|
||||
dispose(t[tensor2]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input2, config3) {
|
||||
const inputHeight = input2.shape[1];
|
||||
const inputWidth = input2.shape[2];
|
||||
const image7 = tidy(() => sub(div(image.resizeBilinear(input2, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image7, config3);
|
||||
dispose(image7);
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
dispose(prediction.box);
|
||||
dispose(prediction.palmLandmarks);
|
||||
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p2 = {};
|
||||
p2.box = slice(t.norm, [index, 0], [1, -1]);
|
||||
p2.slice = slice(t.predictions, [index, 5], [1, 14]);
|
||||
p2.norm = this.normalizeLandmarks(p2.slice, index);
|
||||
p2.palmLandmarks = reshape(p2.norm, [-1, 2]);
|
||||
const box4 = await p2.box.data();
|
||||
const startPoint = box4.slice(0, 2);
|
||||
const endPoint = box4.slice(2, 4);
|
||||
const palmLandmarks = await p2.palmLandmarks.array();
|
||||
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = scaleBoxCoordinates2(hand3, [input2.shape[2] / this.inputSize, input2.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p2).forEach((tensor2) => dispose(p2[tensor2]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor2) => dispose(t[tensor2]));
|
||||
return hands;
|
||||
}
|
||||
};
|
||||
|
@ -79334,7 +79330,7 @@ var HandPipeline = class {
|
|||
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
|
||||
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
|
||||
if (config3.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image7, config3);
|
||||
boxes = await this.handDetector.predict(image7, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipAllowed)
|
||||
|
@ -79927,7 +79923,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
|
|||
var inputSize6 = [[0, 0], [0, 0]];
|
||||
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
|
||||
var faceIndex = 4;
|
||||
var boxExpandFact = 1.6;
|
||||
var boxExpandFact = 1.7;
|
||||
var maxDetectorResolution = 512;
|
||||
var detectorExpandFact = 1.4;
|
||||
var skipped9 = Number.MAX_SAFE_INTEGER;
|
||||
|
@ -81270,7 +81266,7 @@ var options2 = {
|
|||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
font: 'small-caps 14px "Segoe UI"',
|
||||
font: 'small-caps 16px "Segoe UI"',
|
||||
lineHeight: 18,
|
||||
lineWidth: 4,
|
||||
pointSize: 2,
|
||||
|
@ -81319,15 +81315,15 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
function lines(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function lines(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -81336,8 +81332,8 @@ function lines(ctx, points = [], localOptions) {
|
|||
ctx.fill();
|
||||
}
|
||||
}
|
||||
function curves(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function curves(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
@ -83243,6 +83239,24 @@ var Human = class {
|
|||
this.performance.warmup = Math.trunc(t1 - t0);
|
||||
return res;
|
||||
}
|
||||
async profile(input2, userConfig) {
|
||||
const profile2 = await this.tf.profile(() => this.detect(input2, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile2.kernels) {
|
||||
if (kernels[kernel.name])
|
||||
kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else
|
||||
kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
|
||||
kernelArr.sort((a, b) => b.ms - a.ms);
|
||||
kernelArr.length = 20;
|
||||
const res = {};
|
||||
for (const kernel of kernelArr)
|
||||
res[kernel.name] = kernel.ms;
|
||||
return res;
|
||||
}
|
||||
async detect(input2, userConfig) {
|
||||
this.state = "detect";
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -231,7 +231,7 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2e3,
|
||||
skipTime: 1e3,
|
||||
minConfidence: 0.5,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -250,7 +250,7 @@ var config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1e3
|
||||
skipTime: 2e3
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
@ -4645,7 +4645,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
|
|||
var UV7 = VTX7.map((x) => UV468[x]);
|
||||
|
||||
// src/face/facemeshutil.ts
|
||||
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
|
||||
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
|
||||
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
|
||||
var getClampedBox = (box4, input) => box4 ? [
|
||||
|
@ -4816,63 +4815,71 @@ async function load3(config3) {
|
|||
return model3;
|
||||
}
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = tf6.add(boxStarts, anchors);
|
||||
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
|
||||
const centersNormalized = tf6.div(centers, inputSize);
|
||||
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
|
||||
const starts = tf6.sub(centersNormalized, halfBoxSize);
|
||||
const ends = tf6.add(centersNormalized, halfBoxSize);
|
||||
const startNormalized = tf6.mul(starts, inputSize);
|
||||
const endNormalized = tf6.mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t = {};
|
||||
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tf6.add(t.boxStarts, anchors);
|
||||
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = tf6.div(t.centers, inputSize);
|
||||
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
|
||||
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tf6.mul(t.starts, inputSize);
|
||||
t.endNormalized = tf6.mul(t.ends, inputSize);
|
||||
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return boxes;
|
||||
}
|
||||
async function getBoxes(inputImage, config3) {
|
||||
var _a, _b, _c, _d;
|
||||
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return { boxes: [] };
|
||||
const [batch, boxes, scores] = tf6.tidy(() => {
|
||||
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf6.concat([concat512, concat384], 1);
|
||||
batchOut = tf6.squeeze(concat3, 0);
|
||||
} else {
|
||||
batchOut = tf6.squeeze(res);
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await nmsTensor.array();
|
||||
tf6.dispose(nmsTensor);
|
||||
const annotatedBoxes = [];
|
||||
const scoresData = await scores.data();
|
||||
const t = {};
|
||||
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = tf6.div(t.resized, 127.5);
|
||||
t.normalized = tf6.sub(t.div, 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(t.normalized);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
t.concat = tf6.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tf6.squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = tf6.squeeze(res);
|
||||
}
|
||||
tf6.dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf6.sigmoid(t.logits);
|
||||
t.scores = tf6.squeeze(t.sigmoid);
|
||||
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await t.nms.array();
|
||||
const boxes = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
|
||||
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
tf6.dispose(boundingBox);
|
||||
const b = {};
|
||||
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = tf6.squeeze(b.slice);
|
||||
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: await b.startPoint.data(),
|
||||
endPoint: await b.endPoint.data()
|
||||
},
|
||||
landmarks: await b.landmarks.array(),
|
||||
confidence
|
||||
});
|
||||
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
|
||||
}
|
||||
}
|
||||
tf6.dispose(batch);
|
||||
tf6.dispose(boxes);
|
||||
tf6.dispose(scores);
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
|
||||
};
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
||||
// src/body/blazepose.ts
|
||||
|
@ -5438,32 +5445,25 @@ async function predict5(image25, config3, idx, count2) {
|
|||
var _a2, _b2;
|
||||
const obj = [];
|
||||
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
|
||||
const t = {};
|
||||
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
|
||||
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
const [red, green, blue] = tf10.split(resize, 3, 3);
|
||||
tf10.dispose(resize);
|
||||
const redNorm = tf10.mul(red, rgb[0]);
|
||||
const greenNorm = tf10.mul(green, rgb[1]);
|
||||
const blueNorm = tf10.mul(blue, rgb[2]);
|
||||
tf10.dispose(red);
|
||||
tf10.dispose(green);
|
||||
tf10.dispose(blue);
|
||||
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
|
||||
tf10.dispose(redNorm);
|
||||
tf10.dispose(greenNorm);
|
||||
tf10.dispose(blueNorm);
|
||||
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
|
||||
tf10.dispose(grayscale);
|
||||
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
|
||||
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
|
||||
t.redNorm = tf10.mul(t.red, rgb[0]);
|
||||
t.greenNorm = tf10.mul(t.green, rgb[1]);
|
||||
t.blueNorm = tf10.mul(t.blue, rgb[2]);
|
||||
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
|
||||
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
|
||||
lastTime5 = now();
|
||||
const data = await emotionT.data();
|
||||
tf10.dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
|
||||
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf10.dispose(normalize);
|
||||
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
|
||||
}
|
||||
last2[idx] = obj;
|
||||
lastCount2 = count2;
|
||||
|
@ -5624,14 +5624,13 @@ async function predict6(input, config3) {
|
|||
boxCache = [];
|
||||
for (const possible of possibleBoxes.boxes) {
|
||||
const box4 = {
|
||||
startPoint: await possible.box.startPoint.data(),
|
||||
endPoint: await possible.box.endPoint.data(),
|
||||
landmarks: await possible.landmarks.array(),
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence
|
||||
};
|
||||
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped7 = 0;
|
||||
} else {
|
||||
skipped7++;
|
||||
|
@ -5747,17 +5746,13 @@ async function load9(config3) {
|
|||
return model9;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image25 = tf13.tidy(() => {
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(tensor3 instanceof tf13.Tensor))
|
||||
return null;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return null;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
return norm;
|
||||
});
|
||||
return image25;
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return tensor3;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
tf13.dispose(crop2);
|
||||
return norm;
|
||||
}
|
||||
async function predict7(image25, config3, idx, count2) {
|
||||
var _a, _b, _c, _d;
|
||||
|
@ -8891,27 +8886,39 @@ var HandDetector = class {
|
|||
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
|
||||
}
|
||||
normalizeBoxes(boxes) {
|
||||
return tf15.tidy(() => {
|
||||
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return tf15.concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t = {};
|
||||
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
|
||||
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tf15.tidy(() => {
|
||||
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return tf15.mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const t = {};
|
||||
t.batched = this.model.execute(input);
|
||||
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tf15.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tf15.add(t.div, this.anchors[index]);
|
||||
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
async predict(input, config3) {
|
||||
const t = {};
|
||||
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tf15.div(t.resize, 127.5);
|
||||
t.image = tf15.sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image);
|
||||
t.predictions = tf15.squeeze(t.batched);
|
||||
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf15.sigmoid(t.slice);
|
||||
t.scores = tf15.squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
|
@ -8919,32 +8926,21 @@ var HandDetector = class {
|
|||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor3 of Object.keys(t))
|
||||
tf15.dispose(t[tensor3]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image25, config3);
|
||||
tf15.dispose(image25);
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tf15.dispose(prediction.box);
|
||||
tf15.dispose(prediction.palmLandmarks);
|
||||
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p = {};
|
||||
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
|
||||
p.norm = this.normalizeLandmarks(p.slice, index);
|
||||
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
|
||||
const box4 = await p.box.data();
|
||||
const startPoint = box4.slice(0, 2);
|
||||
const endPoint = box4.slice(2, 4);
|
||||
const palmLandmarks = await p.palmLandmarks.array();
|
||||
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return hands;
|
||||
}
|
||||
};
|
||||
|
@ -9024,7 +9020,7 @@ var HandPipeline = class {
|
|||
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
|
||||
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
|
||||
if (config3.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image25, config3);
|
||||
boxes = await this.handDetector.predict(image25, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipAllowed)
|
||||
|
@ -9618,7 +9614,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
|
|||
var inputSize6 = [[0, 0], [0, 0]];
|
||||
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
|
||||
var faceIndex = 4;
|
||||
var boxExpandFact = 1.6;
|
||||
var boxExpandFact = 1.7;
|
||||
var maxDetectorResolution = 512;
|
||||
var detectorExpandFact = 1.4;
|
||||
var skipped9 = Number.MAX_SAFE_INTEGER;
|
||||
|
@ -10971,7 +10967,7 @@ var options2 = {
|
|||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
font: 'small-caps 14px "Segoe UI"',
|
||||
font: 'small-caps 16px "Segoe UI"',
|
||||
lineHeight: 18,
|
||||
lineWidth: 4,
|
||||
pointSize: 2,
|
||||
|
@ -11020,15 +11016,15 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
function lines(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function lines(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -11037,8 +11033,8 @@ function lines(ctx, points = [], localOptions) {
|
|||
ctx.fill();
|
||||
}
|
||||
}
|
||||
function curves(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function curves(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
@ -12948,6 +12944,24 @@ var Human = class {
|
|||
this.performance.warmup = Math.trunc(t1 - t0);
|
||||
return res;
|
||||
}
|
||||
async profile(input, userConfig) {
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile.kernels) {
|
||||
if (kernels[kernel.name])
|
||||
kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else
|
||||
kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
|
||||
kernelArr.sort((a, b) => b.ms - a.ms);
|
||||
kernelArr.length = 20;
|
||||
const res = {};
|
||||
for (const kernel of kernelArr)
|
||||
res[kernel.name] = kernel.ms;
|
||||
return res;
|
||||
}
|
||||
async detect(input, userConfig) {
|
||||
this.state = "detect";
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
|
@ -232,7 +232,7 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2e3,
|
||||
skipTime: 1e3,
|
||||
minConfidence: 0.5,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -251,7 +251,7 @@ var config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1e3
|
||||
skipTime: 2e3
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
@ -4646,7 +4646,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
|
|||
var UV7 = VTX7.map((x) => UV468[x]);
|
||||
|
||||
// src/face/facemeshutil.ts
|
||||
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
|
||||
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
|
||||
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
|
||||
var getClampedBox = (box4, input) => box4 ? [
|
||||
|
@ -4817,63 +4816,71 @@ async function load3(config3) {
|
|||
return model3;
|
||||
}
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = tf6.add(boxStarts, anchors);
|
||||
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
|
||||
const centersNormalized = tf6.div(centers, inputSize);
|
||||
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
|
||||
const starts = tf6.sub(centersNormalized, halfBoxSize);
|
||||
const ends = tf6.add(centersNormalized, halfBoxSize);
|
||||
const startNormalized = tf6.mul(starts, inputSize);
|
||||
const endNormalized = tf6.mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t = {};
|
||||
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tf6.add(t.boxStarts, anchors);
|
||||
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = tf6.div(t.centers, inputSize);
|
||||
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
|
||||
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tf6.mul(t.starts, inputSize);
|
||||
t.endNormalized = tf6.mul(t.ends, inputSize);
|
||||
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return boxes;
|
||||
}
|
||||
async function getBoxes(inputImage, config3) {
|
||||
var _a, _b, _c, _d;
|
||||
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return { boxes: [] };
|
||||
const [batch, boxes, scores] = tf6.tidy(() => {
|
||||
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf6.concat([concat512, concat384], 1);
|
||||
batchOut = tf6.squeeze(concat3, 0);
|
||||
} else {
|
||||
batchOut = tf6.squeeze(res);
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await nmsTensor.array();
|
||||
tf6.dispose(nmsTensor);
|
||||
const annotatedBoxes = [];
|
||||
const scoresData = await scores.data();
|
||||
const t = {};
|
||||
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = tf6.div(t.resized, 127.5);
|
||||
t.normalized = tf6.sub(t.div, 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(t.normalized);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
t.concat = tf6.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tf6.squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = tf6.squeeze(res);
|
||||
}
|
||||
tf6.dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf6.sigmoid(t.logits);
|
||||
t.scores = tf6.squeeze(t.sigmoid);
|
||||
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await t.nms.array();
|
||||
const boxes = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
|
||||
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
tf6.dispose(boundingBox);
|
||||
const b = {};
|
||||
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = tf6.squeeze(b.slice);
|
||||
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: await b.startPoint.data(),
|
||||
endPoint: await b.endPoint.data()
|
||||
},
|
||||
landmarks: await b.landmarks.array(),
|
||||
confidence
|
||||
});
|
||||
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
|
||||
}
|
||||
}
|
||||
tf6.dispose(batch);
|
||||
tf6.dispose(boxes);
|
||||
tf6.dispose(scores);
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
|
||||
};
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
||||
// src/body/blazepose.ts
|
||||
|
@ -5439,32 +5446,25 @@ async function predict5(image25, config3, idx, count2) {
|
|||
var _a2, _b2;
|
||||
const obj = [];
|
||||
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
|
||||
const t = {};
|
||||
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
|
||||
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
const [red, green, blue] = tf10.split(resize, 3, 3);
|
||||
tf10.dispose(resize);
|
||||
const redNorm = tf10.mul(red, rgb[0]);
|
||||
const greenNorm = tf10.mul(green, rgb[1]);
|
||||
const blueNorm = tf10.mul(blue, rgb[2]);
|
||||
tf10.dispose(red);
|
||||
tf10.dispose(green);
|
||||
tf10.dispose(blue);
|
||||
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
|
||||
tf10.dispose(redNorm);
|
||||
tf10.dispose(greenNorm);
|
||||
tf10.dispose(blueNorm);
|
||||
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
|
||||
tf10.dispose(grayscale);
|
||||
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
|
||||
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
|
||||
t.redNorm = tf10.mul(t.red, rgb[0]);
|
||||
t.greenNorm = tf10.mul(t.green, rgb[1]);
|
||||
t.blueNorm = tf10.mul(t.blue, rgb[2]);
|
||||
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
|
||||
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
|
||||
lastTime5 = now();
|
||||
const data = await emotionT.data();
|
||||
tf10.dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
|
||||
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf10.dispose(normalize);
|
||||
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
|
||||
}
|
||||
last2[idx] = obj;
|
||||
lastCount2 = count2;
|
||||
|
@ -5625,14 +5625,13 @@ async function predict6(input, config3) {
|
|||
boxCache = [];
|
||||
for (const possible of possibleBoxes.boxes) {
|
||||
const box4 = {
|
||||
startPoint: await possible.box.startPoint.data(),
|
||||
endPoint: await possible.box.endPoint.data(),
|
||||
landmarks: await possible.landmarks.array(),
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence
|
||||
};
|
||||
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped7 = 0;
|
||||
} else {
|
||||
skipped7++;
|
||||
|
@ -5748,17 +5747,13 @@ async function load9(config3) {
|
|||
return model9;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image25 = tf13.tidy(() => {
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(tensor3 instanceof tf13.Tensor))
|
||||
return null;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return null;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
return norm;
|
||||
});
|
||||
return image25;
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return tensor3;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
tf13.dispose(crop2);
|
||||
return norm;
|
||||
}
|
||||
async function predict7(image25, config3, idx, count2) {
|
||||
var _a, _b, _c, _d;
|
||||
|
@ -8892,27 +8887,39 @@ var HandDetector = class {
|
|||
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
|
||||
}
|
||||
normalizeBoxes(boxes) {
|
||||
return tf15.tidy(() => {
|
||||
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return tf15.concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t = {};
|
||||
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
|
||||
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tf15.tidy(() => {
|
||||
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return tf15.mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const t = {};
|
||||
t.batched = this.model.execute(input);
|
||||
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tf15.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tf15.add(t.div, this.anchors[index]);
|
||||
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
async predict(input, config3) {
|
||||
const t = {};
|
||||
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tf15.div(t.resize, 127.5);
|
||||
t.image = tf15.sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image);
|
||||
t.predictions = tf15.squeeze(t.batched);
|
||||
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf15.sigmoid(t.slice);
|
||||
t.scores = tf15.squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
|
@ -8920,32 +8927,21 @@ var HandDetector = class {
|
|||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor3 of Object.keys(t))
|
||||
tf15.dispose(t[tensor3]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image25, config3);
|
||||
tf15.dispose(image25);
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tf15.dispose(prediction.box);
|
||||
tf15.dispose(prediction.palmLandmarks);
|
||||
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p = {};
|
||||
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
|
||||
p.norm = this.normalizeLandmarks(p.slice, index);
|
||||
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
|
||||
const box4 = await p.box.data();
|
||||
const startPoint = box4.slice(0, 2);
|
||||
const endPoint = box4.slice(2, 4);
|
||||
const palmLandmarks = await p.palmLandmarks.array();
|
||||
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return hands;
|
||||
}
|
||||
};
|
||||
|
@ -9025,7 +9021,7 @@ var HandPipeline = class {
|
|||
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
|
||||
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
|
||||
if (config3.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image25, config3);
|
||||
boxes = await this.handDetector.predict(image25, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipAllowed)
|
||||
|
@ -9619,7 +9615,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
|
|||
var inputSize6 = [[0, 0], [0, 0]];
|
||||
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
|
||||
var faceIndex = 4;
|
||||
var boxExpandFact = 1.6;
|
||||
var boxExpandFact = 1.7;
|
||||
var maxDetectorResolution = 512;
|
||||
var detectorExpandFact = 1.4;
|
||||
var skipped9 = Number.MAX_SAFE_INTEGER;
|
||||
|
@ -10972,7 +10968,7 @@ var options2 = {
|
|||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
font: 'small-caps 14px "Segoe UI"',
|
||||
font: 'small-caps 16px "Segoe UI"',
|
||||
lineHeight: 18,
|
||||
lineWidth: 4,
|
||||
pointSize: 2,
|
||||
|
@ -11021,15 +11017,15 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
function lines(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function lines(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -11038,8 +11034,8 @@ function lines(ctx, points = [], localOptions) {
|
|||
ctx.fill();
|
||||
}
|
||||
}
|
||||
function curves(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function curves(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
@ -12949,6 +12945,24 @@ var Human = class {
|
|||
this.performance.warmup = Math.trunc(t1 - t0);
|
||||
return res;
|
||||
}
|
||||
async profile(input, userConfig) {
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile.kernels) {
|
||||
if (kernels[kernel.name])
|
||||
kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else
|
||||
kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
|
||||
kernelArr.sort((a, b) => b.ms - a.ms);
|
||||
kernelArr.length = 20;
|
||||
const res = {};
|
||||
for (const kernel of kernelArr)
|
||||
res[kernel.name] = kernel.ms;
|
||||
return res;
|
||||
}
|
||||
async detect(input, userConfig) {
|
||||
this.state = "detect";
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
|
@ -231,7 +231,7 @@ var config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2e3,
|
||||
skipTime: 1e3,
|
||||
minConfidence: 0.5,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -250,7 +250,7 @@ var config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1e3
|
||||
skipTime: 2e3
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
@ -4645,7 +4645,6 @@ var UV33 = VTX33.map((x) => UV468[x]);
|
|||
var UV7 = VTX7.map((x) => UV468[x]);
|
||||
|
||||
// src/face/facemeshutil.ts
|
||||
var createBox = (startEndTensor) => ({ startPoint: tf5.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf5.slice(startEndTensor, [0, 2], [-1, 2]) });
|
||||
var getBoxSize = (box4) => [Math.abs(box4.endPoint[0] - box4.startPoint[0]), Math.abs(box4.endPoint[1] - box4.startPoint[1])];
|
||||
var getBoxCenter = (box4) => [box4.startPoint[0] + (box4.endPoint[0] - box4.startPoint[0]) / 2, box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2];
|
||||
var getClampedBox = (box4, input) => box4 ? [
|
||||
|
@ -4816,63 +4815,71 @@ async function load3(config3) {
|
|||
return model3;
|
||||
}
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = tf6.add(boxStarts, anchors);
|
||||
const boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = tf6.div(boxSizes, inputSize);
|
||||
const centersNormalized = tf6.div(centers, inputSize);
|
||||
const halfBoxSize = tf6.div(boxSizesNormalized, 2);
|
||||
const starts = tf6.sub(centersNormalized, halfBoxSize);
|
||||
const ends = tf6.add(centersNormalized, halfBoxSize);
|
||||
const startNormalized = tf6.mul(starts, inputSize);
|
||||
const endNormalized = tf6.mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return tf6.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t = {};
|
||||
t.boxStarts = tf6.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tf6.add(t.boxStarts, anchors);
|
||||
t.boxSizes = tf6.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = tf6.div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = tf6.div(t.centers, inputSize);
|
||||
t.halfBoxSize = tf6.div(t.boxSizesNormalized, 2);
|
||||
t.starts = tf6.sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = tf6.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tf6.mul(t.starts, inputSize);
|
||||
t.endNormalized = tf6.mul(t.ends, inputSize);
|
||||
const boxes = tf6.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return boxes;
|
||||
}
|
||||
async function getBoxes(inputImage, config3) {
|
||||
var _a, _b, _c, _d;
|
||||
if (!inputImage || inputImage["isDisposedInternal"] || inputImage.shape.length !== 4 || inputImage.shape[1] < 1 || inputImage.shape[2] < 1)
|
||||
return { boxes: [] };
|
||||
const [batch, boxes, scores] = tf6.tidy(() => {
|
||||
const resizedImage = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = tf6.sub(tf6.div(resizedImage, 127.5), 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf6.concat([concat512, concat384], 1);
|
||||
batchOut = tf6.squeeze(concat3, 0);
|
||||
} else {
|
||||
batchOut = tf6.squeeze(res);
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = tf6.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf6.squeeze(tf6.sigmoid(logits));
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const nmsTensor = await tf6.image.nonMaxSuppressionAsync(boxes, scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await nmsTensor.array();
|
||||
tf6.dispose(nmsTensor);
|
||||
const annotatedBoxes = [];
|
||||
const scoresData = await scores.data();
|
||||
const t = {};
|
||||
t.resized = tf6.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = tf6.div(t.resized, 127.5);
|
||||
t.normalized = tf6.sub(t.div, 0.5);
|
||||
const res = model3 == null ? void 0 : model3.execute(t.normalized);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = tf6.concat([sorted[0], sorted[2]], 2);
|
||||
t.concat512 = tf6.concat([sorted[1], sorted[3]], 2);
|
||||
t.concat = tf6.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tf6.squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = tf6.squeeze(res);
|
||||
}
|
||||
tf6.dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = tf6.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf6.sigmoid(t.logits);
|
||||
t.scores = tf6.squeeze(t.sigmoid);
|
||||
t.nms = await tf6.image.nonMaxSuppressionAsync(t.boxes, t.scores, ((_a = config3.face.detector) == null ? void 0 : _a.maxDetected) || 0, ((_b = config3.face.detector) == null ? void 0 : _b.iouThreshold) || 0, ((_c = config3.face.detector) == null ? void 0 : _c.minConfidence) || 0);
|
||||
const nms = await t.nms.array();
|
||||
const boxes = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (((_d = config3.face.detector) == null ? void 0 : _d.minConfidence) || 0)) {
|
||||
const boundingBox = tf6.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tf6.tidy(() => tf6.reshape(tf6.squeeze(tf6.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
tf6.dispose(boundingBox);
|
||||
const b = {};
|
||||
b.bbox = tf6.slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = tf6.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = tf6.squeeze(b.slice);
|
||||
b.landmarks = tf6.reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = tf6.slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = tf6.slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: await b.startPoint.data(),
|
||||
endPoint: await b.endPoint.data()
|
||||
},
|
||||
landmarks: await b.landmarks.array(),
|
||||
confidence
|
||||
});
|
||||
Object.keys(b).forEach((tensor3) => tf6.dispose(b[tensor3]));
|
||||
}
|
||||
}
|
||||
tf6.dispose(batch);
|
||||
tf6.dispose(boxes);
|
||||
tf6.dispose(scores);
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize]
|
||||
};
|
||||
Object.keys(t).forEach((tensor3) => tf6.dispose(t[tensor3]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
||||
// src/body/blazepose.ts
|
||||
|
@ -5438,32 +5445,25 @@ async function predict5(image25, config3, idx, count2) {
|
|||
var _a2, _b2;
|
||||
const obj = [];
|
||||
if ((_a2 = config3.face.emotion) == null ? void 0 : _a2.enabled) {
|
||||
const t = {};
|
||||
const inputSize8 = (model6 == null ? void 0 : model6.inputs[0].shape) ? model6.inputs[0].shape[2] : 0;
|
||||
const resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
const [red, green, blue] = tf10.split(resize, 3, 3);
|
||||
tf10.dispose(resize);
|
||||
const redNorm = tf10.mul(red, rgb[0]);
|
||||
const greenNorm = tf10.mul(green, rgb[1]);
|
||||
const blueNorm = tf10.mul(blue, rgb[2]);
|
||||
tf10.dispose(red);
|
||||
tf10.dispose(green);
|
||||
tf10.dispose(blue);
|
||||
const grayscale = tf10.addN([redNorm, greenNorm, blueNorm]);
|
||||
tf10.dispose(redNorm);
|
||||
tf10.dispose(greenNorm);
|
||||
tf10.dispose(blueNorm);
|
||||
const normalize = tf10.tidy(() => tf10.mul(tf10.sub(grayscale, 0.5), 2));
|
||||
tf10.dispose(grayscale);
|
||||
const emotionT = model6 == null ? void 0 : model6.execute(normalize);
|
||||
t.resize = tf10.image.resizeBilinear(image25, [inputSize8, inputSize8], false);
|
||||
[t.red, t.green, t.blue] = tf10.split(t.resize, 3, 3);
|
||||
t.redNorm = tf10.mul(t.red, rgb[0]);
|
||||
t.greenNorm = tf10.mul(t.green, rgb[1]);
|
||||
t.blueNorm = tf10.mul(t.blue, rgb[2]);
|
||||
t.grayscale = tf10.addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = tf10.sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = tf10.mul(t.grayscaleSub, 2);
|
||||
t.emotion = model6 == null ? void 0 : model6.execute(t.grayscaleMul);
|
||||
lastTime5 = now();
|
||||
const data = await emotionT.data();
|
||||
tf10.dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (((_b2 = config3.face.emotion) == null ? void 0 : _b2.minConfidence) || 0))
|
||||
obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf10.dispose(normalize);
|
||||
Object.keys(t).forEach((tensor3) => tf10.dispose(t[tensor3]));
|
||||
}
|
||||
last2[idx] = obj;
|
||||
lastCount2 = count2;
|
||||
|
@ -5624,14 +5624,13 @@ async function predict6(input, config3) {
|
|||
boxCache = [];
|
||||
for (const possible of possibleBoxes.boxes) {
|
||||
const box4 = {
|
||||
startPoint: await possible.box.startPoint.data(),
|
||||
endPoint: await possible.box.endPoint.data(),
|
||||
landmarks: await possible.landmarks.array(),
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence
|
||||
};
|
||||
boxCache.push(squarifyBox(enlargeBox(scaleBoxCoordinates(box4, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tf12.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped7 = 0;
|
||||
} else {
|
||||
skipped7++;
|
||||
|
@ -5747,17 +5746,13 @@ async function load9(config3) {
|
|||
return model9;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image25 = tf13.tidy(() => {
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(tensor3 instanceof tf13.Tensor))
|
||||
return null;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return null;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
return norm;
|
||||
});
|
||||
return image25;
|
||||
const tensor3 = input.image || input.tensor || input;
|
||||
if (!(model9 == null ? void 0 : model9.inputs[0].shape))
|
||||
return tensor3;
|
||||
const crop2 = tf13.image.resizeBilinear(tensor3, [model9.inputs[0].shape[2], model9.inputs[0].shape[1]], false);
|
||||
const norm = tf13.mul(crop2, 255);
|
||||
tf13.dispose(crop2);
|
||||
return norm;
|
||||
}
|
||||
async function predict7(image25, config3, idx, count2) {
|
||||
var _a, _b, _c, _d;
|
||||
|
@ -8891,27 +8886,39 @@ var HandDetector = class {
|
|||
this.doubleInputSizeTensor = tf15.tensor1d([this.inputSize * 2, this.inputSize * 2]);
|
||||
}
|
||||
normalizeBoxes(boxes) {
|
||||
return tf15.tidy(() => {
|
||||
const boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = tf15.add(tf15.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = tf15.div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = tf15.mul(tf15.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = tf15.mul(tf15.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return tf15.concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t = {};
|
||||
t.boxOffsets = tf15.slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = tf15.slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = tf15.div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = tf15.add(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = tf15.div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = tf15.sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = tf15.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tf15.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tf15.mul(t.add, this.inputSizeTensor);
|
||||
const res = tf15.concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tf15.tidy(() => {
|
||||
const landmarks = tf15.add(tf15.div(tf15.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return tf15.mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
}
|
||||
async getBoxes(input, config3) {
|
||||
const t = {};
|
||||
t.batched = this.model.execute(input);
|
||||
t.reshape = tf15.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tf15.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tf15.add(t.div, this.anchors[index]);
|
||||
const res = tf15.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return res;
|
||||
}
|
||||
async predict(input, config3) {
|
||||
const t = {};
|
||||
t.resize = tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tf15.div(t.resize, 127.5);
|
||||
t.image = tf15.sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image);
|
||||
t.predictions = tf15.squeeze(t.batched);
|
||||
t.scores = tf15.tidy(() => tf15.squeeze(tf15.sigmoid(tf15.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = tf15.slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf15.sigmoid(t.slice);
|
||||
t.scores = tf15.squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf15.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
|
@ -8919,32 +8926,21 @@ var HandDetector = class {
|
|||
const nms = await t.nms.array();
|
||||
const hands = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf15.tidy(() => tf15.reshape(this.normalizeLandmarks(tf15.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor3 of Object.keys(t))
|
||||
tf15.dispose(t[tensor3]);
|
||||
return hands;
|
||||
}
|
||||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image25 = tf15.tidy(() => tf15.sub(tf15.div(tf15.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image25, config3);
|
||||
tf15.dispose(image25);
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tf15.dispose(prediction.box);
|
||||
tf15.dispose(prediction.palmLandmarks);
|
||||
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p = {};
|
||||
p.box = tf15.slice(t.norm, [index, 0], [1, -1]);
|
||||
p.slice = tf15.slice(t.predictions, [index, 5], [1, 14]);
|
||||
p.norm = this.normalizeLandmarks(p.slice, index);
|
||||
p.palmLandmarks = tf15.reshape(p.norm, [-1, 2]);
|
||||
const box4 = await p.box.data();
|
||||
const startPoint = box4.slice(0, 2);
|
||||
const endPoint = box4.slice(2, 4);
|
||||
const palmLandmarks = await p.palmLandmarks.array();
|
||||
const hand3 = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = scaleBoxCoordinates2(hand3, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p).forEach((tensor3) => tf15.dispose(p[tensor3]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor3) => tf15.dispose(t[tensor3]));
|
||||
return hands;
|
||||
}
|
||||
};
|
||||
|
@ -9024,7 +9020,7 @@ var HandPipeline = class {
|
|||
const skipTime = (config3.hand.skipTime || 0) > now() - lastTime8;
|
||||
const skipFrame = this.skipped < (config3.hand.skipFrames || 0);
|
||||
if (config3.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image25, config3);
|
||||
boxes = await this.handDetector.predict(image25, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipAllowed)
|
||||
|
@ -9618,7 +9614,7 @@ var modelOutputNodes = ["StatefulPartitionedCall/Postprocessor/Slice", "Stateful
|
|||
var inputSize6 = [[0, 0], [0, 0]];
|
||||
var classes = ["hand", "fist", "pinch", "point", "face", "tip", "pinchtip"];
|
||||
var faceIndex = 4;
|
||||
var boxExpandFact = 1.6;
|
||||
var boxExpandFact = 1.7;
|
||||
var maxDetectorResolution = 512;
|
||||
var detectorExpandFact = 1.4;
|
||||
var skipped9 = Number.MAX_SAFE_INTEGER;
|
||||
|
@ -10971,7 +10967,7 @@ var options2 = {
|
|||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
font: 'small-caps 14px "Segoe UI"',
|
||||
font: 'small-caps 16px "Segoe UI"',
|
||||
lineHeight: 18,
|
||||
lineWidth: 4,
|
||||
pointSize: 2,
|
||||
|
@ -11020,15 +11016,15 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
function lines(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function lines(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + 2 * z}, ${127.5 - 2 * z}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -11037,8 +11033,8 @@ function lines(ctx, points = [], localOptions) {
|
|||
ctx.fill();
|
||||
}
|
||||
}
|
||||
function curves(ctx, points = [], localOptions) {
|
||||
if (points === void 0 || points.length === 0)
|
||||
function curves(ctx, points, localOptions) {
|
||||
if (points.length < 2)
|
||||
return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
@ -12948,6 +12944,24 @@ var Human = class {
|
|||
this.performance.warmup = Math.trunc(t1 - t0);
|
||||
return res;
|
||||
}
|
||||
async profile(input, userConfig) {
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile.kernels) {
|
||||
if (kernels[kernel.name])
|
||||
kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else
|
||||
kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] }));
|
||||
kernelArr.sort((a, b) => b.ms - a.ms);
|
||||
kernelArr.length = 20;
|
||||
const res = {};
|
||||
for (const kernel of kernelArr)
|
||||
res[kernel.name] = kernel.ms;
|
||||
return res;
|
||||
}
|
||||
async detect(input, userConfig) {
|
||||
this.state = "detect";
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
|
@ -339,7 +339,7 @@ const config: Config = {
|
|||
enabled: true,
|
||||
rotation: true,
|
||||
skipFrames: 99,
|
||||
skipTime: 2000,
|
||||
skipTime: 1000,
|
||||
minConfidence: 0.50,
|
||||
iouThreshold: 0.2,
|
||||
maxDetected: -1,
|
||||
|
@ -358,7 +358,7 @@ const config: Config = {
|
|||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
skipFrames: 99,
|
||||
skipTime: 1000,
|
||||
skipTime: 2000,
|
||||
},
|
||||
segmentation: {
|
||||
enabled: false,
|
||||
|
|
|
@ -9,6 +9,7 @@ import * as util from './facemeshutil';
|
|||
import type { Config } from '../config';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
import type { Point } from '../result';
|
||||
|
||||
const keypointsCount = 6;
|
||||
let model: GraphModel | null;
|
||||
|
@ -34,63 +35,72 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
}
|
||||
|
||||
function decodeBounds(boxOutputs) {
|
||||
const boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
const centers = tf.add(boxStarts, anchors);
|
||||
const boxSizes = tf.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
const boxSizesNormalized = tf.div(boxSizes, inputSize);
|
||||
const centersNormalized = tf.div(centers, inputSize);
|
||||
const halfBoxSize = tf.div(boxSizesNormalized, 2);
|
||||
const starts = tf.sub(centersNormalized, halfBoxSize);
|
||||
const ends = tf.add(centersNormalized, halfBoxSize);
|
||||
const startNormalized = tf.mul(starts, inputSize);
|
||||
const endNormalized = tf.mul(ends, inputSize);
|
||||
const concatAxis = 1;
|
||||
return tf.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tf.add(t.boxStarts, anchors);
|
||||
t.boxSizes = tf.slice(boxOutputs, [0, 3], [-1, 2]);
|
||||
t.boxSizesNormalized = tf.div(t.boxSizes, inputSize);
|
||||
t.centersNormalized = tf.div(t.centers, inputSize);
|
||||
t.halfBoxSize = tf.div(t.boxSizesNormalized, 2);
|
||||
t.starts = tf.sub(t.centersNormalized, t.halfBoxSize);
|
||||
t.ends = tf.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tf.mul(t.starts, inputSize);
|
||||
t.endNormalized = tf.mul(t.ends, inputSize);
|
||||
const boxes = tf.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return boxes;
|
||||
}
|
||||
|
||||
export async function getBoxes(inputImage: Tensor, config: Config) {
|
||||
// sanity check on input
|
||||
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return { boxes: [] };
|
||||
const [batch, boxes, scores] = tf.tidy(() => {
|
||||
const resizedImage = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
const normalizedImage = tf.sub(tf.div(resizedImage, 127.5), 0.5);
|
||||
const res = model?.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
|
||||
const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
|
||||
const concat = tf.concat([concat512, concat384], 1);
|
||||
batchOut = tf.squeeze(concat, 0);
|
||||
} else {
|
||||
batchOut = tf.squeeze(res); // when using tfhub model
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut);
|
||||
const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf.squeeze(tf.sigmoid(logits)); // inside tf.tidy
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const t: Record<string, Tensor> = {};
|
||||
|
||||
const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
|
||||
const nms = await nmsTensor.array();
|
||||
tf.dispose(nmsTensor);
|
||||
const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: [number, number] | undefined, confidence: number }> = [];
|
||||
const scoresData = await scores.data();
|
||||
t.resized = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
|
||||
t.div = tf.div(t.resized, 127.5);
|
||||
t.normalized = tf.sub(t.div, 0.5);
|
||||
const res = model?.execute(t.normalized) as Tensor[];
|
||||
if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
t.concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
|
||||
t.concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
|
||||
t.concat = tf.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tf.squeeze(t.concat, 0);
|
||||
} else {
|
||||
t.batch = tf.squeeze(res); // when using tfhub model
|
||||
}
|
||||
tf.dispose(res);
|
||||
t.boxes = decodeBounds(t.batch);
|
||||
t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf.sigmoid(t.logits);
|
||||
t.scores = tf.squeeze(t.sigmoid);
|
||||
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
|
||||
const nms = await t.nms.array() as number[];
|
||||
const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = [];
|
||||
const scores = await t.scores.data();
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scoresData[nms[i]];
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > (config.face.detector?.minConfidence || 0)) {
|
||||
const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const landmarks = tf.tidy(() => tf.reshape(tf.squeeze(tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: util.createBox(boundingBox), landmarks, anchor: anchorsData[nms[i]], confidence });
|
||||
tf.dispose(boundingBox);
|
||||
const b: Record<string, Tensor> = {};
|
||||
b.bbox = tf.slice(t.boxes, [nms[i], 0], [1, -1]);
|
||||
b.slice = tf.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
|
||||
b.squeeze = tf.squeeze(b.slice);
|
||||
b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
|
||||
b.startPoint = tf.slice(b.bbox, [0, 0], [-1, 2]);
|
||||
b.endPoint = tf.slice(b.bbox, [0, 2], [-1, 2]);
|
||||
boxes.push({
|
||||
box: {
|
||||
startPoint: (await b.startPoint.data()) as unknown as Point,
|
||||
endPoint: (await b.endPoint.data()) as unknown as Point,
|
||||
},
|
||||
landmarks: (await b.landmarks.array()) as Point[],
|
||||
confidence,
|
||||
});
|
||||
Object.keys(b).forEach((tensor) => tf.dispose(b[tensor]));
|
||||
}
|
||||
}
|
||||
tf.dispose(batch);
|
||||
tf.dispose(boxes);
|
||||
tf.dispose(scores);
|
||||
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize],
|
||||
};
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
|
||||
}
|
||||
|
|
|
@ -37,14 +37,13 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
|
|||
boxCache = []; // empty cache
|
||||
for (const possible of possibleBoxes.boxes) { // extract data from detector
|
||||
const box: BoxCache = {
|
||||
startPoint: await possible.box.startPoint.data() as unknown as Point,
|
||||
endPoint: await possible.box.endPoint.data() as unknown as Point,
|
||||
landmarks: await possible.landmarks.array() as Array<Point>,
|
||||
startPoint: possible.box.startPoint,
|
||||
endPoint: possible.box.endPoint,
|
||||
landmarks: possible.landmarks,
|
||||
confidence: possible.confidence,
|
||||
};
|
||||
boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped = 0;
|
||||
} else {
|
||||
skipped++;
|
||||
|
|
|
@ -37,57 +37,50 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
}
|
||||
|
||||
export function enhance(input): Tensor {
|
||||
const image = tf.tidy(() => {
|
||||
// input received from detector is already normalized to 0..1
|
||||
// input is also assumed to be straightened
|
||||
const tensor = input.image || input.tensor || input;
|
||||
if (!(tensor instanceof tf.Tensor)) return null;
|
||||
// do a tight crop of image and resize it to fit the model
|
||||
if (!model?.inputs[0].shape) return null; // model has no shape so no point continuing
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
/*
|
||||
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
const crop = (tensor.shape.length === 3)
|
||||
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
|
||||
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
*/
|
||||
/*
|
||||
// just resize to fit the embedding model instead of cropping
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
*/
|
||||
const tensor = (input.image || input.tensor || input) as Tensor; // input received from detector is already normalized to 0..1, input is also assumed to be straightened
|
||||
if (!model?.inputs[0].shape) return tensor; // model has no shape so no point continuing
|
||||
// do a tight crop of image and resize it to fit the model
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
/*
|
||||
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
const crop = (tensor.shape.length === 3)
|
||||
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
|
||||
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
*/
|
||||
/*
|
||||
// just resize to fit the embedding model instead of cropping
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
*/
|
||||
|
||||
/*
|
||||
// convert to black&white to avoid colorization impact
|
||||
const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
const [red, green, blue] = tf.split(crop, 3, 3);
|
||||
const redNorm = tf.mul(red, rgb[0]);
|
||||
const greenNorm = tf.mul(green, rgb[1]);
|
||||
const blueNorm = tf.mul(blue, rgb[2]);
|
||||
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
|
||||
const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4);
|
||||
*/
|
||||
/*
|
||||
// convert to black&white to avoid colorization impact
|
||||
const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
const [red, green, blue] = tf.split(crop, 3, 3);
|
||||
const redNorm = tf.mul(red, rgb[0]);
|
||||
const greenNorm = tf.mul(green, rgb[1]);
|
||||
const blueNorm = tf.mul(blue, rgb[2]);
|
||||
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
|
||||
const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4);
|
||||
*/
|
||||
|
||||
/*
|
||||
// increase image pseudo-contrast 100%
|
||||
// (or do it per-channel so mean is done on each channel)
|
||||
// (or calculate histogram and do it based on histogram)
|
||||
const mean = merge.mean();
|
||||
const factor = 2;
|
||||
const contrast = merge.sub(mean).mul(factor).add(mean);
|
||||
*/
|
||||
/*
|
||||
// increase image pseudo-contrast 100%
|
||||
// (or do it per-channel so mean is done on each channel)
|
||||
// (or calculate histogram and do it based on histogram)
|
||||
const mean = merge.mean();
|
||||
const factor = 2;
|
||||
const contrast = merge.sub(mean).mul(factor).add(mean);
|
||||
*/
|
||||
|
||||
/*
|
||||
// normalize brightness from 0..1
|
||||
// silly way of creating pseudo-hdr of image
|
||||
const darken = crop.sub(crop.min());
|
||||
const lighten = darken.div(darken.max());
|
||||
*/
|
||||
|
||||
const norm = tf.mul(crop, 255);
|
||||
|
||||
return norm;
|
||||
});
|
||||
return image;
|
||||
/*
|
||||
// normalize brightness from 0..1
|
||||
// silly way of creating pseudo-hdr of image
|
||||
const darken = crop.sub(crop.min());
|
||||
const lighten = darken.div(darken.max());
|
||||
*/
|
||||
const norm = tf.mul(crop, 255);
|
||||
tf.dispose(crop);
|
||||
return norm;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx, count) {
|
||||
|
|
|
@ -126,7 +126,7 @@ export async function augmentIris(rawCoords, face, config, meshSize) {
|
|||
tf.dispose(rightEyeCrop);
|
||||
const eyePredictions = model.execute(combined) as Tensor;
|
||||
tf.dispose(combined);
|
||||
const eyePredictionsData = await eyePredictions.data(); // inside tf.tidy
|
||||
const eyePredictionsData = await eyePredictions.data();
|
||||
tf.dispose(eyePredictions);
|
||||
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
|
||||
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
|
||||
|
|
|
@ -43,35 +43,27 @@ export async function predict(image: Tensor, config: Config, idx, count) {
|
|||
return new Promise(async (resolve) => {
|
||||
const obj: Array<{ score: number, emotion: string }> = [];
|
||||
if (config.face.emotion?.enabled) {
|
||||
const t: Record<string, Tensor> = {};
|
||||
const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0;
|
||||
const resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false);
|
||||
t.resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false);
|
||||
// const box = [[0.15, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
// const resize = tf.image.cropAndResize(image, box, [0], [inputSize, inputSize]);
|
||||
|
||||
const [red, green, blue] = tf.split(resize, 3, 3);
|
||||
tf.dispose(resize);
|
||||
[t.red, t.green, t.blue] = tf.split(t.resize, 3, 3);
|
||||
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
const redNorm = tf.mul(red, rgb[0]);
|
||||
const greenNorm = tf.mul(green, rgb[1]);
|
||||
const blueNorm = tf.mul(blue, rgb[2]);
|
||||
tf.dispose(red);
|
||||
tf.dispose(green);
|
||||
tf.dispose(blue);
|
||||
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
|
||||
tf.dispose(redNorm);
|
||||
tf.dispose(greenNorm);
|
||||
tf.dispose(blueNorm);
|
||||
const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
|
||||
tf.dispose(grayscale);
|
||||
const emotionT = model?.execute(normalize) as Tensor; // result is already in range 0..1, no need for additional activation
|
||||
t.redNorm = tf.mul(t.red, rgb[0]);
|
||||
t.greenNorm = tf.mul(t.green, rgb[1]);
|
||||
t.blueNorm = tf.mul(t.blue, rgb[2]);
|
||||
t.grayscale = tf.addN([t.redNorm, t.greenNorm, t.blueNorm]);
|
||||
t.grayscaleSub = tf.sub(t.grayscale, 0.5);
|
||||
t.grayscaleMul = tf.mul(t.grayscaleSub, 2);
|
||||
t.emotion = model?.execute(t.grayscaleMul) as Tensor; // result is already in range 0..1, no need for additional activation
|
||||
lastTime = now();
|
||||
const data = await emotionT.data();
|
||||
tf.dispose(emotionT);
|
||||
const data = await t.emotion.data();
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
tf.dispose(normalize);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
}
|
||||
last[idx] = obj;
|
||||
lastCount = count;
|
||||
|
|
|
@ -7,6 +7,7 @@ import * as tf from '../../dist/tfjs.esm.js';
|
|||
import * as util from './handposeutil';
|
||||
import * as anchors from './handposeanchors';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Point } from '../result';
|
||||
|
||||
export class HandDetector {
|
||||
model: GraphModel;
|
||||
|
@ -26,62 +27,64 @@ export class HandDetector {
|
|||
}
|
||||
|
||||
normalizeBoxes(boxes) {
|
||||
return tf.tidy(() => {
|
||||
const boxOffsets = tf.slice(boxes, [0, 0], [-1, 2]);
|
||||
const boxSizes = tf.slice(boxes, [0, 2], [-1, 2]);
|
||||
const boxCenterPoints = tf.add(tf.div(boxOffsets, this.inputSizeTensor), this.anchorsTensor);
|
||||
const halfBoxSizes = tf.div(boxSizes, this.doubleInputSizeTensor);
|
||||
const startPoints = tf.mul(tf.sub(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
const endPoints = tf.mul(tf.add(boxCenterPoints, halfBoxSizes), this.inputSizeTensor);
|
||||
return tf.concat2d([startPoints, endPoints], 1);
|
||||
});
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.boxOffsets = tf.slice(boxes, [0, 0], [-1, 2]);
|
||||
t.boxSizes = tf.slice(boxes, [0, 2], [-1, 2]);
|
||||
t.div = tf.div(t.boxOffsets, this.inputSizeTensor);
|
||||
t.boxCenterPoints = tf.add(t.div, this.anchorsTensor);
|
||||
t.halfBoxSizes = tf.div(t.boxSizes, this.doubleInputSizeTensor);
|
||||
t.sub = tf.sub(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.startPoints = tf.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tf.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tf.mul(t.add, this.inputSizeTensor);
|
||||
const res = tf.concat2d([t.startPoints, t.endPoints], 1);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return res;
|
||||
}
|
||||
|
||||
normalizeLandmarks(rawPalmLandmarks, index) {
|
||||
return tf.tidy(() => {
|
||||
const landmarks = tf.add(tf.div(tf.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
|
||||
return tf.mul(landmarks, this.inputSizeTensor);
|
||||
});
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.reshape = tf.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tf.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tf.add(t.div, this.anchors[index]);
|
||||
const res = tf.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return res;
|
||||
}
|
||||
|
||||
async getBoxes(input, config) {
|
||||
async predict(input, config): Promise<{ startPoint: Point; endPoint: Point, palmLandmarks: Point[]; confidence: number }[]> {
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.batched = this.model.execute(input) as Tensor;
|
||||
t.resize = tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tf.div(t.resize, 127.5);
|
||||
t.image = tf.sub(t.div, 1);
|
||||
t.batched = this.model.execute(t.image) as Tensor;
|
||||
t.predictions = tf.squeeze(t.batched);
|
||||
t.scores = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(t.predictions, [0, 0], [-1, 1]))));
|
||||
t.slice = tf.slice(t.predictions, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf.sigmoid(t.slice);
|
||||
t.scores = tf.squeeze(t.sigmoid);
|
||||
const scores = await t.scores.data();
|
||||
t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
// box detection is flaky so we look for 3x boxes than we need results
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 3 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
|
||||
const nms = await t.nms.array() as Array<number>;
|
||||
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
|
||||
const hands: Array<{ startPoint: Point; endPoint: Point; palmLandmarks: Point[]; confidence: number }> = [];
|
||||
for (const index of nms) {
|
||||
const palmBox = tf.slice(t.norm, [index, 0], [1, -1]);
|
||||
const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(tf.slice(t.predictions, [index, 5], [1, 14]), index), [-1, 2]));
|
||||
hands.push({ box: palmBox, palmLandmarks, confidence: scores[index] });
|
||||
}
|
||||
for (const tensor of Object.keys(t)) tf.dispose(t[tensor]); // dispose all
|
||||
return hands;
|
||||
}
|
||||
|
||||
async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image = tf.tidy(() => tf.sub(tf.div(tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
|
||||
const predictions = await this.getBoxes(image, config);
|
||||
tf.dispose(image);
|
||||
const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
|
||||
if (!predictions || predictions.length === 0) return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = await prediction.box.data();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tf.dispose(prediction.box);
|
||||
tf.dispose(prediction.palmLandmarks);
|
||||
hands.push(util.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
const p: Record<string, Tensor> = {};
|
||||
p.box = tf.slice(t.norm, [index, 0], [1, -1]);
|
||||
p.slice = tf.slice(t.predictions, [index, 5], [1, 14]);
|
||||
p.norm = this.normalizeLandmarks(p.slice, index);
|
||||
p.palmLandmarks = tf.reshape(p.norm, [-1, 2]);
|
||||
const box = await p.box.data();
|
||||
const startPoint = box.slice(0, 2) as unknown as Point;
|
||||
const endPoint = box.slice(2, 4) as unknown as Point;
|
||||
const palmLandmarks = await p.palmLandmarks.array();
|
||||
const hand = { startPoint, endPoint, palmLandmarks, confidence: scores[index] };
|
||||
const scaled = util.scaleBoxCoordinates(hand, [input.shape[2] / this.inputSize, input.shape[1] / this.inputSize]);
|
||||
hands.push(scaled);
|
||||
Object.keys(p).forEach((tensor) => tf.dispose(p[tensor]));
|
||||
}
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return hands;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ import type * as detector from './handposedetector';
|
|||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
import { now } from '../util/util';
|
||||
import type { Point } from '../result';
|
||||
|
||||
const palmBoxEnlargeFactor = 5; // default 3
|
||||
const handBoxEnlargeFactor = 1.65; // default 1.65
|
||||
|
@ -21,7 +22,7 @@ export class HandPipeline {
|
|||
handDetector: detector.HandDetector;
|
||||
handPoseModel: GraphModel;
|
||||
inputSize: number;
|
||||
storedBoxes: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number } | null>;
|
||||
storedBoxes: Array<{ startPoint: Point; endPoint: Point; palmLandmarks: Point[]; confidence: number } | null>;
|
||||
skipped: number;
|
||||
detectedHands: number;
|
||||
|
||||
|
@ -93,7 +94,7 @@ export class HandPipeline {
|
|||
const skipTime = (config.hand.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = this.skipped < (config.hand.skipFrames || 0);
|
||||
if (config.skipAllowed && skipTime && skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image, config);
|
||||
boxes = await this.handDetector.predict(image, config);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config.skipAllowed) this.skipped++;
|
||||
|
@ -105,7 +106,7 @@ export class HandPipeline {
|
|||
// for (const possible of boxes) this.storedBoxes.push(possible);
|
||||
if (this.storedBoxes.length > 0) useFreshBox = true;
|
||||
}
|
||||
const hands: Array<{ landmarks: number[], confidence: number, boxConfidence: number, fingerConfidence: number, box: { topLeft: number[], bottomRight: number[] } }> = [];
|
||||
const hands: Array<{ landmarks: Point[], confidence: number, boxConfidence: number, fingerConfidence: number, box: { topLeft: Point, bottomRight: Point } }> = [];
|
||||
|
||||
// go through working set of boxes
|
||||
for (let i = 0; i < this.storedBoxes.length; i++) {
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { Point } from '../result';
|
||||
|
||||
export function getBoxSize(box) {
|
||||
return [
|
||||
|
@ -27,8 +28,8 @@ export function cutBoxFromImageAndResize(box, image, cropSize) {
|
|||
}
|
||||
|
||||
export function scaleBoxCoordinates(box, factor) {
|
||||
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
|
||||
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
|
||||
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]] as Point;
|
||||
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]] as Point;
|
||||
const palmLandmarks = box.palmLandmarks.map((coord) => {
|
||||
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
|
||||
return scaledCoord;
|
||||
|
@ -40,8 +41,8 @@ export function enlargeBox(box, factor = 1.5) {
|
|||
const center = getBoxCenter(box);
|
||||
const size = getBoxSize(box);
|
||||
const newHalfSize = [factor * size[0] / 2, factor * size[1] / 2];
|
||||
const startPoint = [center[0] - newHalfSize[0], center[1] - newHalfSize[1]];
|
||||
const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]];
|
||||
const startPoint = [center[0] - newHalfSize[0], center[1] - newHalfSize[1]] as Point;
|
||||
const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]] as Point;
|
||||
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
|
||||
}
|
||||
|
||||
|
@ -50,8 +51,8 @@ export function squarifyBox(box) {
|
|||
const size = getBoxSize(box);
|
||||
const maxEdge = Math.max(...size);
|
||||
const halfSize = maxEdge / 2;
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize] as Point;
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize] as Point;
|
||||
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
|
||||
}
|
||||
|
||||
|
@ -61,8 +62,8 @@ export function shiftBox(box, shiftFactor) {
|
|||
box.endPoint[1] - box.startPoint[1],
|
||||
];
|
||||
const shiftVector = [boxSize[0] * shiftFactor[0], boxSize[1] * shiftFactor[1]];
|
||||
const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]];
|
||||
const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]];
|
||||
const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]] as Point;
|
||||
const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]] as Point;
|
||||
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
|
||||
}
|
||||
|
||||
|
|
|
@ -24,7 +24,7 @@ const inputSize = [[0, 0], [0, 0]];
|
|||
const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip'];
|
||||
const faceIndex = 4;
|
||||
|
||||
const boxExpandFact = 1.6;
|
||||
const boxExpandFact = 1.7;
|
||||
const maxDetectorResolution = 512;
|
||||
const detectorExpandFact = 1.4;
|
||||
|
||||
|
|
20
src/human.ts
20
src/human.ts
|
@ -347,6 +347,26 @@ export class Human {
|
|||
return res;
|
||||
}
|
||||
|
||||
/** Run detect with tensorflow profiling
|
||||
* - result object will contain total exeuction time information for top-20 kernels
|
||||
* - actual detection object can be accessed via `human.result`
|
||||
*/
|
||||
async profile(input: Input, userConfig?: Partial<Config>): Promise<Record<string, number>> {
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels = {};
|
||||
for (const kernel of profile.kernels) { // sum kernel time values per kernel
|
||||
if (kernels[kernel.name]) kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
}
|
||||
const kernelArr: Array<{ name, ms }> = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ name: key[0], ms: key[1] })); // convert to array
|
||||
kernelArr.sort((a, b) => b.ms - a.ms); // sort
|
||||
kernelArr.length = 20; // crop
|
||||
const res: Record<string, number> = {};
|
||||
for (const kernel of kernelArr) res[kernel.name] = kernel.ms; // create perf objects
|
||||
return res;
|
||||
}
|
||||
|
||||
/** Main detection method
|
||||
* - Analyze configuration: {@link Config}
|
||||
* - Pre-process input: {@link Input}
|
||||
|
|
|
@ -52,7 +52,7 @@ export const options: DrawOptions = {
|
|||
color: <string>'rgba(173, 216, 230, 0.6)', // 'lightblue' with light alpha channel
|
||||
labelColor: <string>'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel
|
||||
shadowColor: <string>'black',
|
||||
font: <string>'small-caps 14px "Segoe UI"',
|
||||
font: <string>'small-caps 16px "Segoe UI"',
|
||||
lineHeight: <number>18,
|
||||
lineWidth: <number>4,
|
||||
pointSize: <number>2,
|
||||
|
@ -106,14 +106,14 @@ function rect(ctx: CanvasRenderingContext2D, x, y, width, height, localOptions)
|
|||
ctx.stroke();
|
||||
}
|
||||
|
||||
function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
function lines(ctx: CanvasRenderingContext2D, points: Point[], localOptions) {
|
||||
if (points.length < 2) return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z !== 0 ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
@ -123,8 +123,8 @@ function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions
|
|||
}
|
||||
}
|
||||
|
||||
function curves(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
function curves(ctx: CanvasRenderingContext2D, points: Point[], localOptions) {
|
||||
if (points.length < 2) return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
return;
|
||||
|
|
3066
test/build.log
3066
test/build.log
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue