ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix unregistered ops in tfjs

pull/193/head
Vladimir Mandic 2021-07-29 16:06:03 -04:00
parent bbc45d3c39
commit fc424b176d
33 changed files with 169022 additions and 3675 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
demo/face3d/face3d.js
View File

@ -16,7 +16,7 @@ const userConfig = {
profile: false, profile: false,
warmup: 'full', warmup: 'full',
modelBasePath: '../../models/', modelBasePath: '../../models/',
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.7.0/dist/', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.8.0/dist/',
filter: { enabled: false }, filter: { enabled: false },
face: { enabled: true, face: { enabled: true,
detector: { rotation: false, maxDetected: 1 }, detector: { rotation: false, maxDetected: 1 },

8
demo/facematch/facematch.js
View File

@ -14,7 +14,7 @@ const userConfig = {
warmup: 'none', warmup: 'none',
debug: true, debug: true,
modelBasePath: '../../models/', modelBasePath: '../../models/',
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.7.0/dist/', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.8.0/dist/',
face: { face: {
enabled: true, enabled: true,
detector: { rotation: true, return: true }, detector: { rotation: true, return: true },
@ -75,10 +75,10 @@ async function analyze(face) {
navigator.clipboard.writeText(`{"name":"unknown", "source":"${face.fileName}", "embedding":[${embedding}]},`); navigator.clipboard.writeText(`{"name":"unknown", "source":"${face.fileName}", "embedding":[${embedding}]},`);
if (enhanced) { if (enhanced) {
const c = document.getElementById('orig'); const c = document.getElementById('orig');
const squeeze = enhanced.squeeze().div(255); const squeeze = human.tf.div(human.tf.squeeze(enhanced), 255);
await human.tf.browser.toPixels(squeeze, c); await human.tf.browser.toPixels(squeeze, c);
enhanced.dispose(); human.tf.dispose(enhanced);
squeeze.dispose(); human.tf.dispose(squeeze);
const ctx = c.getContext('2d'); const ctx = c.getContext('2d');
ctx.font = 'small-caps 0.4rem "Lato"'; ctx.font = 'small-caps 0.4rem "Lato"';
ctx.fillStyle = 'rgba(255, 255, 255, 1)'; ctx.fillStyle = 'rgba(255, 255, 255, 1)';

18
demo/index.js
View File

@ -30,7 +30,7 @@ let human;
let userConfig = { let userConfig = {
warmup: 'none', warmup: 'none',
backend: 'humangl', backend: 'humangl',
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.7.0/dist/', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.8.0/dist/',
/* /*
async: false, async: false,
cacheSensitivity: 0, cacheSensitivity: 0,
@ -169,10 +169,10 @@ function status(msg) {
} }
const compare = { enabled: false, original: null }; const compare = { enabled: false, original: null };
async function calcSimmilariry(result) { async function calcSimmilarity(result) {
document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none'; document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none';
if (!compare.enabled) return; if (!compare.enabled) return;
if (!result || !result.face || !result.face[0].embedding) return; if (!result || !result.face || !result.face[0] || !result.face[0].embedding) return;
if (!(result.face.length > 0) || (result.face[0].embedding.length <= 64)) return; if (!(result.face.length > 0) || (result.face[0].embedding.length <= 64)) return;
if (!compare.original) { if (!compare.original) {
compare.original = result; compare.original = result;
@ -181,12 +181,12 @@ async function calcSimmilariry(result) {
const enhanced = human.enhance(result.face[0]); const enhanced = human.enhance(result.face[0]);
if (enhanced) { if (enhanced) {
const c = document.getElementById('orig'); const c = document.getElementById('orig');
const squeeze = enhanced.squeeze(); const squeeze = human.tf.squeeze(enhanced);
const norm = squeeze.div(255); const norm = human.tf.div(squeeze, 255);
human.tf.browser.toPixels(norm, c); human.tf.browser.toPixels(norm, c);
enhanced.dispose(); human.tf.dispose(enhanced);
squeeze.dispose(); human.tf.dispose(squeeze);
norm.dispose(); human.tf.dispose(norm);
} }
} else { } else {
document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200); document.getElementById('compare-canvas').getContext('2d').drawImage(compare.original.canvas, 0, 0, 200, 200);
@ -246,7 +246,7 @@ async function drawResults(input) {
*/ */
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const person = result.persons; // explicitly invoke person getter const person = result.persons; // explicitly invoke person getter
await calcSimmilariry(result); await calcSimmilarity(result);
// update log // update log
const engine = human.tf.engine(); const engine = human.tf.engine();

2
demo/nodejs/node-multiprocess-worker.js
View File

@ -53,7 +53,7 @@ async function detect(img) {
process.send({ image: img, detected: result }); // send results back to main process.send({ image: img, detected: result }); // send results back to main
process.send({ ready: true }); // send signal back to main that this worker is now idle and ready for next image process.send({ ready: true }); // send signal back to main that this worker is now idle and ready for next image
} }
tensor.dispose(); tf.dispose(tensor);
} }
async function main() { async function main() {

4
demo/nodejs/node-video.js
View File

@ -66,13 +66,13 @@ async function process(jpegBuffer) {
busy = true; busy = true;
const decoded = tf.node.decodeJpeg(jpegBuffer, 3); // decode jpeg buffer to raw tensor const decoded = tf.node.decodeJpeg(jpegBuffer, 3); // decode jpeg buffer to raw tensor
const tensor = tf.expandDims(decoded, 0); // almost all tf models use first dimension as batch number so we add it const tensor = tf.expandDims(decoded, 0); // almost all tf models use first dimension as batch number so we add it
decoded.dispose(); tf.dispose(decoded);
log.state('input frame:', ++count, 'size:', jpegBuffer.length, 'decoded shape:', tensor.shape); log.state('input frame:', ++count, 'size:', jpegBuffer.length, 'decoded shape:', tensor.shape);
const res = await human.detect(tensor); const res = await human.detect(tensor);
log.data('gesture', JSON.stringify(res.gesture)); log.data('gesture', JSON.stringify(res.gesture));
// do processing here // do processing here
tensor.dispose(); // must dispose tensor tf.dispose(tensor); // must dispose tensor
busy = false; busy = false;
} }

11017
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

80465
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

80493
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

166
dist/human.node-gpu.js vendored
View File

@ -498,7 +498,7 @@ var BlazeFaceModel = class {
return null; return null;
const [batch, boxes, scores] = tf3.tidy(() => { const [batch, boxes, scores] = tf3.tidy(() => {
const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]); const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]);
const normalizedImage = resizedImage.div(127.5).sub(0.5); const normalizedImage = tf3.sub(tf3.div(resizedImage, 127.5), 0.5);
const res = this.model.execute(normalizedImage); const res = this.model.execute(normalizedImage);
let batchOut; let batchOut;
if (Array.isArray(res)) { if (Array.isArray(res)) {
@ -506,33 +506,33 @@ var BlazeFaceModel = class {
const concat384 = tf3.concat([sorted[0], sorted[2]], 2); const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
const concat512 = tf3.concat([sorted[1], sorted[3]], 2); const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
const concat3 = tf3.concat([concat512, concat384], 1); const concat3 = tf3.concat([concat512, concat384], 1);
batchOut = concat3.squeeze(0); batchOut = tf3.squeeze(concat3, 0);
} else { } else {
batchOut = tf3.squeeze(res); batchOut = tf3.squeeze(res);
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync();
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence); const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
const nms = nmsTensor.arraySync(); const nms = nmsTensor.arraySync();
nmsTensor.dispose(); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scores[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
boundingBox.dispose(); tf3.dispose(boundingBox);
const anchor = this.anchorsData[nms[i]]; const anchor = this.anchorsData[nms[i]];
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1])); const landmarks = tf3.tidy(() => tf3.reshape(tf3.squeeze(tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence }); annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
} }
} }
batch.dispose(); tf3.dispose(batch);
boxes.dispose(); tf3.dispose(boxes);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3977,9 +3977,9 @@ var Pipeline = class {
} }
if (detector && detector.boxes) { if (detector && detector.boxes) {
detector.boxes.forEach((prediction) => { detector.boxes.forEach((prediction) => {
prediction.box.startPoint.dispose(); tf4.dispose(prediction.box.startPoint);
prediction.box.endPoint.dispose(); tf4.dispose(prediction.box.endPoint);
prediction.landmarks.dispose(); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => {
@ -3994,16 +3994,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4055,9 +4055,9 @@ var Pipeline = class {
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]); angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint }); const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
} }
const prediction = { const prediction = {
mesh, mesh,
@ -4122,7 +4122,7 @@ async function predict(input, config3) {
tensor: prediction.image tensor: prediction.image
}); });
if (prediction.coords) if (prediction.coords)
prediction.coords.dispose(); tf5.dispose(prediction.coords);
} }
return results; return results;
} }
@ -4210,7 +4210,7 @@ function enhance(input) {
if (!model.inputs[0].shape) if (!model.inputs[0].shape)
return null; return null;
const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
const norm = crop.mul(255); const norm = tf6.mul(crop, 255);
return norm; return norm;
}); });
return image18; return image18;
@ -4244,7 +4244,7 @@ async function predict2(image18, config3, idx, count2) {
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.genderScore = Math.min(0.99, confidence); obj.genderScore = Math.min(0.99, confidence);
} }
const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0]; const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = resT.find((t) => t.shape[1] === 100).dataSync();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
@ -4288,19 +4288,19 @@ async function predict3(image18, config3, idx, count2) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false); const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false);
const [red, green, blue] = tf7.split(resize, 3, 3); const [red, green, blue] = tf7.split(resize, 3, 3);
resize.dispose(); tf7.dispose(resize);
const redNorm = tf7.mul(red, rgb[0]); const redNorm = tf7.mul(red, rgb[0]);
const greenNorm = tf7.mul(green, rgb[1]); const greenNorm = tf7.mul(green, rgb[1]);
const blueNorm = tf7.mul(blue, rgb[2]); const blueNorm = tf7.mul(blue, rgb[2]);
red.dispose(); tf7.dispose(red);
green.dispose(); tf7.dispose(green);
blue.dispose(); tf7.dispose(blue);
const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]); const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]);
redNorm.dispose(); tf7.dispose(redNorm);
greenNorm.dispose(); tf7.dispose(greenNorm);
blueNorm.dispose(); tf7.dispose(blueNorm);
const normalize = tf7.tidy(() => grayscale.sub(0.5).mul(2)); const normalize = tf7.tidy(() => tf7.mul(tf7.sub(grayscale, 0.5), 2));
grayscale.dispose(); tf7.dispose(grayscale);
const obj = []; const obj = [];
if (config3.face.emotion.enabled) { if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize); const emotionT = await model2.predict(normalize);
@ -4312,7 +4312,7 @@ async function predict3(image18, config3, idx, count2) {
} }
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); tf7.dispose(normalize);
last2[idx] = obj; last2[idx] = obj;
lastCount2 = count2; lastCount2 = count2;
resolve(obj); resolve(obj);
@ -4636,7 +4636,7 @@ async function predict4(input, config3) {
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]); const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]);
const normalized = resized.toFloat().div(127.5).sub(1); const normalized = tf8.sub(tf8.div(tf8.cast(resized, "float32"), 127.5), 1);
const results = model3.execute(normalized, poseNetOutputs); const results = model3.execute(normalized, poseNetOutputs);
const results3d = results.map((y) => tf8.squeeze(y, [0])); const results3d = results.map((y) => tf8.squeeze(y, [0]));
results3d[1] = results3d[1].sigmoid(); results3d[1] = results3d[1].sigmoid();
@ -4644,7 +4644,7 @@ async function predict4(input, config3) {
}); });
const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer())); const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer()));
for (const t of res) for (const t of res)
t.dispose(); tf8.dispose(t);
const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence); const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence);
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
@ -7693,43 +7693,43 @@ var HandDetector = class {
} }
normalizeLandmarks(rawPalmLandmarks, index) { normalizeLandmarks(rawPalmLandmarks, index) {
return tf10.tidy(() => { return tf10.tidy(() => {
const landmarks = tf10.add(tf10.div(rawPalmLandmarks.reshape([-1, 7, 2]), this.inputSizeTensor), this.anchors[index]); const landmarks = tf10.add(tf10.div(tf10.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf10.mul(landmarks, this.inputSizeTensor); return tf10.mul(landmarks, this.inputSizeTensor);
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched); const predictions = tf10.squeeze(batched);
batched.dispose(); tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])).squeeze()); const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync(); const scores = scoresT.dataSync();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose(); tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = filteredT.arraySync(); const filtered = filteredT.arraySync();
scoresT.dispose(); tf10.dispose(scoresT);
filteredT.dispose(); tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) { if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, index).reshape([-1, 2])); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
rawPalmLandmarks.dispose(); tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] }); hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
} }
} }
predictions.dispose(); tf10.dispose(predictions);
boxes.dispose(); tf10.dispose(boxes);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image18 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image18 = tf10.tidy(() => tf10.sub(tf10.div(tf10.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image18, config3); const predictions = await this.getBoxes(image18, config3);
image18.dispose(); tf10.dispose(image18);
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -7738,8 +7738,8 @@ var HandDetector = class {
const startPoint = boxes.slice(0, 2); const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4); const endPoint = boxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.box.dispose(); tf10.dispose(prediction.box);
prediction.palmLandmarks.dispose(); tf10.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
} }
return hands; return hands;
@ -7902,18 +7902,18 @@ var HandPipeline = class {
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
const handImage = croppedInput.div(255); const handImage = tf11.div(croppedInput, 255);
croppedInput.dispose(); tf11.dispose(croppedInput);
rotatedImage.dispose(); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
handImage.dispose(); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = confidenceT.dataSync()[0];
confidenceT.dispose(); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync(); const rawCoords = keypointsReshaped.arraySync();
keypoints3.dispose(); tf11.dispose(keypoints3);
keypointsReshaped.dispose(); tf11.dispose(keypointsReshaped);
const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords3); const nextBoundingBox = this.getBoxForHandLandmarks(coords3);
this.storedBoxes[i] = { ...nextBoundingBox, confidence }; this.storedBoxes[i] = { ...nextBoundingBox, confidence };
@ -7926,7 +7926,7 @@ var HandPipeline = class {
} else { } else {
this.storedBoxes[i] = null; this.storedBoxes[i] = null;
} }
keypoints3.dispose(); tf11.dispose(keypoints3);
} else { } else {
const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor); const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor);
const result = { const result = {
@ -8131,11 +8131,11 @@ async function predict6(image18, config3) {
const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 }; const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 };
const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false); const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); tf13.dispose(resize);
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || []; const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
resT.forEach((t) => t.dispose()); resT.forEach((t) => tf13.dispose(t));
normalize.dispose(); tf13.dispose(normalize);
const keypoints3 = []; const keypoints3 = [];
const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper; const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper;
const depth = 5; const depth = 5;
@ -8223,7 +8223,7 @@ async function predict7(image18, config3) {
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model5.predict(tensor2); resT = await model5.predict(tensor2);
tensor2.dispose(); tf14.dispose(tensor2);
if (resT) { if (resT) {
keypoints.length = 0; keypoints.length = 0;
const squeeze7 = resT.squeeze(); const squeeze7 = resT.squeeze();
@ -8301,13 +8301,13 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast2 = tf15.cast(resize, "int32"); const cast5 = tf15.cast(resize, "int32");
return cast2; return cast5;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model6.predict(tensor2); resT = await model6.predict(tensor2);
tensor2.dispose(); tf15.dispose(tensor2);
if (resT) { if (resT) {
keypoints2.length = 0; keypoints2.length = 0;
const res = resT.arraySync(); const res = resT.arraySync();
@ -8529,14 +8529,14 @@ async function predict9(image18, config3) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image18.shape[2], image18.shape[1]]; const outputSize = [image18.shape[2], image18.shape[1]];
const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false); const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false);
const norm = resize.div(255); const norm = tf16.div(resize, 255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); tf16.dispose(norm);
resize.dispose(); tf16.dispose(resize);
let objectT; let objectT;
if (config3.object.enabled) if (config3.object.enabled)
objectT = await model7.predict(transpose); objectT = await model7.predict(transpose);
transpose.dispose(); tf16.dispose(transpose);
const obj = await process2(objectT, model7.inputSize, outputSize, config3); const obj = await process2(objectT, model7.inputSize, outputSize, config3);
last3 = obj; last3 = obj;
resolve(obj); resolve(obj);
@ -8569,20 +8569,20 @@ async function process3(res, inputSize, outputShape, config3) {
const results = []; const results = [];
const detections = res.arraySync(); const detections = res.arraySync();
const squeezeT = tf17.squeeze(res); const squeezeT = tf17.squeeze(res);
res.dispose(); tf17.dispose(res);
const arr = tf17.split(squeezeT, 6, 1); const arr = tf17.split(squeezeT, 6, 1);
squeezeT.dispose(); tf17.dispose(squeezeT);
const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1); const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze(); const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze(); const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze(); const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose()); arr.forEach((t) => tf17.dispose(t));
const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence); const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose(); tf17.dispose(boxesT);
scoresT.dispose(); tf17.dispose(scoresT);
classesT.dispose(); tf17.dispose(classesT);
const nms = nmsT.dataSync(); const nms = nmsT.dataSync();
nmsT.dispose(); tf17.dispose(nmsT);
let i = 0; let i = 0;
for (const id of nms) { for (const id of nms) {
const score3 = Math.trunc(100 * detections[0][id][4]) / 100; const score3 = Math.trunc(100 * detections[0][id][4]) / 100;
@ -8618,7 +8618,7 @@ async function predict10(input, config3) {
const outputSize = [input.shape[2], input.shape[1]]; const outputSize = [input.shape[2], input.shape[1]];
const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]); const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]);
const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null; const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null;
resize.dispose(); tf17.dispose(resize);
const obj = await process3(objectT, model8.inputSize, outputSize, config3); const obj = await process3(objectT, model8.inputSize, outputSize, config3);
last4 = obj; last4 = obj;
resolve(obj); resolve(obj);
@ -9467,10 +9467,10 @@ function process4(input, config3) {
pixels = tf18.browser ? tf18.browser.fromPixels(data) : null; pixels = tf18.browser ? tf18.browser.fromPixels(data) : null;
} }
if (pixels) { if (pixels) {
const casted = pixels.toFloat(); const casted = tf18.cast(pixels, "float32");
tensor2 = casted.expandDims(0); tensor2 = tf18.expandDims(casted, 0);
pixels.dispose(); tf18.dispose(pixels);
casted.dispose(); tf18.dispose(casted);
} }
} }
const canvas2 = config3.filter.return ? outCanvas : null; const canvas2 = config3.filter.return ? outCanvas : null;
@ -9500,7 +9500,7 @@ async function predict11(input) {
if (!model9 || !model9.inputs[0].shape) if (!model9 || !model9.inputs[0].shape)
return null; return null;
const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false); const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false);
const norm = resizeInput.div(255); const norm = tf19.div(resizeInput, 255);
const res = model9.predict(norm); const res = model9.predict(norm);
tf19.dispose(resizeInput); tf19.dispose(resizeInput);
tf19.dispose(norm); tf19.dispose(norm);
@ -9509,13 +9509,13 @@ async function predict11(input) {
if (squeeze7.shape[2] === 2) { if (squeeze7.shape[2] === 2) {
const softmax = squeeze7.softmax(); const softmax = squeeze7.softmax();
const [bg, fg] = tf19.unstack(softmax, 2); const [bg, fg] = tf19.unstack(softmax, 2);
const expand = fg.expandDims(2); const expand = tf19.expandDims(fg, 2);
const pad = expand.expandDims(0); const pad = tf19.expandDims(expand, 0);
tf19.dispose(softmax); tf19.dispose(softmax);
tf19.dispose(bg); tf19.dispose(bg);
tf19.dispose(fg); tf19.dispose(fg);
const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]); const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]);
resizeOutput = crop.squeeze(0); resizeOutput = tf19.squeeze(crop, 0);
tf19.dispose(crop); tf19.dispose(crop);
tf19.dispose(expand); tf19.dispose(expand);
tf19.dispose(pad); tf19.dispose(pad);
@ -11419,7 +11419,7 @@ var Human = class {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return false; return false;
const resizeFact = 32; const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync(); const reducedData = reduced.dataSync();
let sum = 0; let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++) for (let i = 0; i < reducedData.length / 3; i++)
@ -11610,7 +11610,7 @@ var Human = class {
if (elapsedTime > 0) if (elapsedTime > 0)
this.performance.segmentation = elapsedTime; this.performance.segmentation = elapsedTime;
if (process6.canvas) { if (process6.canvas) {
process6.tensor.dispose(); tf21.dispose(process6.tensor);
process6 = process4(process6.canvas, this.config); process6 = process4(process6.canvas, this.config);
} }
this.analyze("End Segmentation:"); this.analyze("End Segmentation:");

166
dist/human.node-wasm.js vendored
View File

@ -499,7 +499,7 @@ var BlazeFaceModel = class {
return null; return null;
const [batch, boxes, scores] = tf3.tidy(() => { const [batch, boxes, scores] = tf3.tidy(() => {
const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]); const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]);
const normalizedImage = resizedImage.div(127.5).sub(0.5); const normalizedImage = tf3.sub(tf3.div(resizedImage, 127.5), 0.5);
const res = this.model.execute(normalizedImage); const res = this.model.execute(normalizedImage);
let batchOut; let batchOut;
if (Array.isArray(res)) { if (Array.isArray(res)) {
@ -507,33 +507,33 @@ var BlazeFaceModel = class {
const concat384 = tf3.concat([sorted[0], sorted[2]], 2); const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
const concat512 = tf3.concat([sorted[1], sorted[3]], 2); const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
const concat3 = tf3.concat([concat512, concat384], 1); const concat3 = tf3.concat([concat512, concat384], 1);
batchOut = concat3.squeeze(0); batchOut = tf3.squeeze(concat3, 0);
} else { } else {
batchOut = tf3.squeeze(res); batchOut = tf3.squeeze(res);
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync();
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence); const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
const nms = nmsTensor.arraySync(); const nms = nmsTensor.arraySync();
nmsTensor.dispose(); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scores[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
boundingBox.dispose(); tf3.dispose(boundingBox);
const anchor = this.anchorsData[nms[i]]; const anchor = this.anchorsData[nms[i]];
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1])); const landmarks = tf3.tidy(() => tf3.reshape(tf3.squeeze(tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence }); annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
} }
} }
batch.dispose(); tf3.dispose(batch);
boxes.dispose(); tf3.dispose(boxes);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3978,9 +3978,9 @@ var Pipeline = class {
} }
if (detector && detector.boxes) { if (detector && detector.boxes) {
detector.boxes.forEach((prediction) => { detector.boxes.forEach((prediction) => {
prediction.box.startPoint.dispose(); tf4.dispose(prediction.box.startPoint);
prediction.box.endPoint.dispose(); tf4.dispose(prediction.box.endPoint);
prediction.landmarks.dispose(); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => {
@ -3995,16 +3995,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4056,9 +4056,9 @@ var Pipeline = class {
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]); angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint }); const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
} }
const prediction = { const prediction = {
mesh, mesh,
@ -4123,7 +4123,7 @@ async function predict(input, config3) {
tensor: prediction.image tensor: prediction.image
}); });
if (prediction.coords) if (prediction.coords)
prediction.coords.dispose(); tf5.dispose(prediction.coords);
} }
return results; return results;
} }
@ -4211,7 +4211,7 @@ function enhance(input) {
if (!model.inputs[0].shape) if (!model.inputs[0].shape)
return null; return null;
const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
const norm = crop.mul(255); const norm = tf6.mul(crop, 255);
return norm; return norm;
}); });
return image18; return image18;
@ -4245,7 +4245,7 @@ async function predict2(image18, config3, idx, count2) {
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.genderScore = Math.min(0.99, confidence); obj.genderScore = Math.min(0.99, confidence);
} }
const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0]; const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = resT.find((t) => t.shape[1] === 100).dataSync();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
@ -4289,19 +4289,19 @@ async function predict3(image18, config3, idx, count2) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false); const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false);
const [red, green, blue] = tf7.split(resize, 3, 3); const [red, green, blue] = tf7.split(resize, 3, 3);
resize.dispose(); tf7.dispose(resize);
const redNorm = tf7.mul(red, rgb[0]); const redNorm = tf7.mul(red, rgb[0]);
const greenNorm = tf7.mul(green, rgb[1]); const greenNorm = tf7.mul(green, rgb[1]);
const blueNorm = tf7.mul(blue, rgb[2]); const blueNorm = tf7.mul(blue, rgb[2]);
red.dispose(); tf7.dispose(red);
green.dispose(); tf7.dispose(green);
blue.dispose(); tf7.dispose(blue);
const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]); const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]);
redNorm.dispose(); tf7.dispose(redNorm);
greenNorm.dispose(); tf7.dispose(greenNorm);
blueNorm.dispose(); tf7.dispose(blueNorm);
const normalize = tf7.tidy(() => grayscale.sub(0.5).mul(2)); const normalize = tf7.tidy(() => tf7.mul(tf7.sub(grayscale, 0.5), 2));
grayscale.dispose(); tf7.dispose(grayscale);
const obj = []; const obj = [];
if (config3.face.emotion.enabled) { if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize); const emotionT = await model2.predict(normalize);
@ -4313,7 +4313,7 @@ async function predict3(image18, config3, idx, count2) {
} }
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); tf7.dispose(normalize);
last2[idx] = obj; last2[idx] = obj;
lastCount2 = count2; lastCount2 = count2;
resolve(obj); resolve(obj);
@ -4637,7 +4637,7 @@ async function predict4(input, config3) {
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]); const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]);
const normalized = resized.toFloat().div(127.5).sub(1); const normalized = tf8.sub(tf8.div(tf8.cast(resized, "float32"), 127.5), 1);
const results = model3.execute(normalized, poseNetOutputs); const results = model3.execute(normalized, poseNetOutputs);
const results3d = results.map((y) => tf8.squeeze(y, [0])); const results3d = results.map((y) => tf8.squeeze(y, [0]));
results3d[1] = results3d[1].sigmoid(); results3d[1] = results3d[1].sigmoid();
@ -4645,7 +4645,7 @@ async function predict4(input, config3) {
}); });
const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer())); const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer()));
for (const t of res) for (const t of res)
t.dispose(); tf8.dispose(t);
const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence); const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence);
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
@ -7694,43 +7694,43 @@ var HandDetector = class {
} }
normalizeLandmarks(rawPalmLandmarks, index) { normalizeLandmarks(rawPalmLandmarks, index) {
return tf10.tidy(() => { return tf10.tidy(() => {
const landmarks = tf10.add(tf10.div(rawPalmLandmarks.reshape([-1, 7, 2]), this.inputSizeTensor), this.anchors[index]); const landmarks = tf10.add(tf10.div(tf10.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf10.mul(landmarks, this.inputSizeTensor); return tf10.mul(landmarks, this.inputSizeTensor);
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched); const predictions = tf10.squeeze(batched);
batched.dispose(); tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])).squeeze()); const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync(); const scores = scoresT.dataSync();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose(); tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = filteredT.arraySync(); const filtered = filteredT.arraySync();
scoresT.dispose(); tf10.dispose(scoresT);
filteredT.dispose(); tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) { if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, index).reshape([-1, 2])); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
rawPalmLandmarks.dispose(); tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] }); hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
} }
} }
predictions.dispose(); tf10.dispose(predictions);
boxes.dispose(); tf10.dispose(boxes);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image18 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image18 = tf10.tidy(() => tf10.sub(tf10.div(tf10.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image18, config3); const predictions = await this.getBoxes(image18, config3);
image18.dispose(); tf10.dispose(image18);
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -7739,8 +7739,8 @@ var HandDetector = class {
const startPoint = boxes.slice(0, 2); const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4); const endPoint = boxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.box.dispose(); tf10.dispose(prediction.box);
prediction.palmLandmarks.dispose(); tf10.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
} }
return hands; return hands;
@ -7903,18 +7903,18 @@ var HandPipeline = class {
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
const handImage = croppedInput.div(255); const handImage = tf11.div(croppedInput, 255);
croppedInput.dispose(); tf11.dispose(croppedInput);
rotatedImage.dispose(); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
handImage.dispose(); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = confidenceT.dataSync()[0];
confidenceT.dispose(); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync(); const rawCoords = keypointsReshaped.arraySync();
keypoints3.dispose(); tf11.dispose(keypoints3);
keypointsReshaped.dispose(); tf11.dispose(keypointsReshaped);
const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords3); const nextBoundingBox = this.getBoxForHandLandmarks(coords3);
this.storedBoxes[i] = { ...nextBoundingBox, confidence }; this.storedBoxes[i] = { ...nextBoundingBox, confidence };
@ -7927,7 +7927,7 @@ var HandPipeline = class {
} else { } else {
this.storedBoxes[i] = null; this.storedBoxes[i] = null;
} }
keypoints3.dispose(); tf11.dispose(keypoints3);
} else { } else {
const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor); const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor);
const result = { const result = {
@ -8132,11 +8132,11 @@ async function predict6(image18, config3) {
const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 }; const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 };
const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false); const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); tf13.dispose(resize);
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || []; const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
resT.forEach((t) => t.dispose()); resT.forEach((t) => tf13.dispose(t));
normalize.dispose(); tf13.dispose(normalize);
const keypoints3 = []; const keypoints3 = [];
const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper; const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper;
const depth = 5; const depth = 5;
@ -8224,7 +8224,7 @@ async function predict7(image18, config3) {
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model5.predict(tensor2); resT = await model5.predict(tensor2);
tensor2.dispose(); tf14.dispose(tensor2);
if (resT) { if (resT) {
keypoints.length = 0; keypoints.length = 0;
const squeeze7 = resT.squeeze(); const squeeze7 = resT.squeeze();
@ -8302,13 +8302,13 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast2 = tf15.cast(resize, "int32"); const cast5 = tf15.cast(resize, "int32");
return cast2; return cast5;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model6.predict(tensor2); resT = await model6.predict(tensor2);
tensor2.dispose(); tf15.dispose(tensor2);
if (resT) { if (resT) {
keypoints2.length = 0; keypoints2.length = 0;
const res = resT.arraySync(); const res = resT.arraySync();
@ -8530,14 +8530,14 @@ async function predict9(image18, config3) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image18.shape[2], image18.shape[1]]; const outputSize = [image18.shape[2], image18.shape[1]];
const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false); const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false);
const norm = resize.div(255); const norm = tf16.div(resize, 255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); tf16.dispose(norm);
resize.dispose(); tf16.dispose(resize);
let objectT; let objectT;
if (config3.object.enabled) if (config3.object.enabled)
objectT = await model7.predict(transpose); objectT = await model7.predict(transpose);
transpose.dispose(); tf16.dispose(transpose);
const obj = await process2(objectT, model7.inputSize, outputSize, config3); const obj = await process2(objectT, model7.inputSize, outputSize, config3);
last3 = obj; last3 = obj;
resolve(obj); resolve(obj);
@ -8570,20 +8570,20 @@ async function process3(res, inputSize, outputShape, config3) {
const results = []; const results = [];
const detections = res.arraySync(); const detections = res.arraySync();
const squeezeT = tf17.squeeze(res); const squeezeT = tf17.squeeze(res);
res.dispose(); tf17.dispose(res);
const arr = tf17.split(squeezeT, 6, 1); const arr = tf17.split(squeezeT, 6, 1);
squeezeT.dispose(); tf17.dispose(squeezeT);
const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1); const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze(); const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze(); const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze(); const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose()); arr.forEach((t) => tf17.dispose(t));
const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence); const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose(); tf17.dispose(boxesT);
scoresT.dispose(); tf17.dispose(scoresT);
classesT.dispose(); tf17.dispose(classesT);
const nms = nmsT.dataSync(); const nms = nmsT.dataSync();
nmsT.dispose(); tf17.dispose(nmsT);
let i = 0; let i = 0;
for (const id of nms) { for (const id of nms) {
const score3 = Math.trunc(100 * detections[0][id][4]) / 100; const score3 = Math.trunc(100 * detections[0][id][4]) / 100;
@ -8619,7 +8619,7 @@ async function predict10(input, config3) {
const outputSize = [input.shape[2], input.shape[1]]; const outputSize = [input.shape[2], input.shape[1]];
const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]); const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]);
const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null; const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null;
resize.dispose(); tf17.dispose(resize);
const obj = await process3(objectT, model8.inputSize, outputSize, config3); const obj = await process3(objectT, model8.inputSize, outputSize, config3);
last4 = obj; last4 = obj;
resolve(obj); resolve(obj);
@ -9468,10 +9468,10 @@ function process4(input, config3) {
pixels = tf18.browser ? tf18.browser.fromPixels(data) : null; pixels = tf18.browser ? tf18.browser.fromPixels(data) : null;
} }
if (pixels) { if (pixels) {
const casted = pixels.toFloat(); const casted = tf18.cast(pixels, "float32");
tensor2 = casted.expandDims(0); tensor2 = tf18.expandDims(casted, 0);
pixels.dispose(); tf18.dispose(pixels);
casted.dispose(); tf18.dispose(casted);
} }
} }
const canvas2 = config3.filter.return ? outCanvas : null; const canvas2 = config3.filter.return ? outCanvas : null;
@ -9501,7 +9501,7 @@ async function predict11(input) {
if (!model9 || !model9.inputs[0].shape) if (!model9 || !model9.inputs[0].shape)
return null; return null;
const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false); const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false);
const norm = resizeInput.div(255); const norm = tf19.div(resizeInput, 255);
const res = model9.predict(norm); const res = model9.predict(norm);
tf19.dispose(resizeInput); tf19.dispose(resizeInput);
tf19.dispose(norm); tf19.dispose(norm);
@ -9510,13 +9510,13 @@ async function predict11(input) {
if (squeeze7.shape[2] === 2) { if (squeeze7.shape[2] === 2) {
const softmax = squeeze7.softmax(); const softmax = squeeze7.softmax();
const [bg, fg] = tf19.unstack(softmax, 2); const [bg, fg] = tf19.unstack(softmax, 2);
const expand = fg.expandDims(2); const expand = tf19.expandDims(fg, 2);
const pad = expand.expandDims(0); const pad = tf19.expandDims(expand, 0);
tf19.dispose(softmax); tf19.dispose(softmax);
tf19.dispose(bg); tf19.dispose(bg);
tf19.dispose(fg); tf19.dispose(fg);
const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]); const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]);
resizeOutput = crop.squeeze(0); resizeOutput = tf19.squeeze(crop, 0);
tf19.dispose(crop); tf19.dispose(crop);
tf19.dispose(expand); tf19.dispose(expand);
tf19.dispose(pad); tf19.dispose(pad);
@ -11420,7 +11420,7 @@ var Human = class {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return false; return false;
const resizeFact = 32; const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync(); const reducedData = reduced.dataSync();
let sum = 0; let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++) for (let i = 0; i < reducedData.length / 3; i++)
@ -11611,7 +11611,7 @@ var Human = class {
if (elapsedTime > 0) if (elapsedTime > 0)
this.performance.segmentation = elapsedTime; this.performance.segmentation = elapsedTime;
if (process6.canvas) { if (process6.canvas) {
process6.tensor.dispose(); tf21.dispose(process6.tensor);
process6 = process4(process6.canvas, this.config); process6 = process4(process6.canvas, this.config);
} }
this.analyze("End Segmentation:"); this.analyze("End Segmentation:");

166
dist/human.node.js vendored
View File

@ -498,7 +498,7 @@ var BlazeFaceModel = class {
return null; return null;
const [batch, boxes, scores] = tf3.tidy(() => { const [batch, boxes, scores] = tf3.tidy(() => {
const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]); const resizedImage = tf3.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]);
const normalizedImage = resizedImage.div(127.5).sub(0.5); const normalizedImage = tf3.sub(tf3.div(resizedImage, 127.5), 0.5);
const res = this.model.execute(normalizedImage); const res = this.model.execute(normalizedImage);
let batchOut; let batchOut;
if (Array.isArray(res)) { if (Array.isArray(res)) {
@ -506,33 +506,33 @@ var BlazeFaceModel = class {
const concat384 = tf3.concat([sorted[0], sorted[2]], 2); const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
const concat512 = tf3.concat([sorted[1], sorted[3]], 2); const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
const concat3 = tf3.concat([concat512, concat384], 1); const concat3 = tf3.concat([concat512, concat384], 1);
batchOut = concat3.squeeze(0); batchOut = tf3.squeeze(concat3, 0);
} else { } else {
batchOut = tf3.squeeze(res); batchOut = tf3.squeeze(res);
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync();
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence); const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
const nms = nmsTensor.arraySync(); const nms = nmsTensor.arraySync();
nmsTensor.dispose(); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scores[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
boundingBox.dispose(); tf3.dispose(boundingBox);
const anchor = this.anchorsData[nms[i]]; const anchor = this.anchorsData[nms[i]];
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1])); const landmarks = tf3.tidy(() => tf3.reshape(tf3.squeeze(tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence }); annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
} }
} }
batch.dispose(); tf3.dispose(batch);
boxes.dispose(); tf3.dispose(boxes);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3977,9 +3977,9 @@ var Pipeline = class {
} }
if (detector && detector.boxes) { if (detector && detector.boxes) {
detector.boxes.forEach((prediction) => { detector.boxes.forEach((prediction) => {
prediction.box.startPoint.dispose(); tf4.dispose(prediction.box.startPoint);
prediction.box.endPoint.dispose(); tf4.dispose(prediction.box.endPoint);
prediction.landmarks.dispose(); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => {
@ -3994,16 +3994,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255);
else else
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4055,9 +4055,9 @@ var Pipeline = class {
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]); angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint }); const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
} }
const prediction = { const prediction = {
mesh, mesh,
@ -4122,7 +4122,7 @@ async function predict(input, config3) {
tensor: prediction.image tensor: prediction.image
}); });
if (prediction.coords) if (prediction.coords)
prediction.coords.dispose(); tf5.dispose(prediction.coords);
} }
return results; return results;
} }
@ -4210,7 +4210,7 @@ function enhance(input) {
if (!model.inputs[0].shape) if (!model.inputs[0].shape)
return null; return null;
const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); const crop = tensor2.shape.length === 3 ? tf6.image.cropAndResize(tf6.expandDims(tensor2, 0), box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) : tf6.image.cropAndResize(tensor2, box6, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
const norm = crop.mul(255); const norm = tf6.mul(crop, 255);
return norm; return norm;
}); });
return image18; return image18;
@ -4244,7 +4244,7 @@ async function predict2(image18, config3, idx, count2) {
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.genderScore = Math.min(0.99, confidence); obj.genderScore = Math.min(0.99, confidence);
} }
const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0]; const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = resT.find((t) => t.shape[1] === 100).dataSync();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
@ -4288,19 +4288,19 @@ async function predict3(image18, config3, idx, count2) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false); const resize = tf7.image.resizeBilinear(image18, [model2.inputs[0].shape[2], model2.inputs[0].shape[1]], false);
const [red, green, blue] = tf7.split(resize, 3, 3); const [red, green, blue] = tf7.split(resize, 3, 3);
resize.dispose(); tf7.dispose(resize);
const redNorm = tf7.mul(red, rgb[0]); const redNorm = tf7.mul(red, rgb[0]);
const greenNorm = tf7.mul(green, rgb[1]); const greenNorm = tf7.mul(green, rgb[1]);
const blueNorm = tf7.mul(blue, rgb[2]); const blueNorm = tf7.mul(blue, rgb[2]);
red.dispose(); tf7.dispose(red);
green.dispose(); tf7.dispose(green);
blue.dispose(); tf7.dispose(blue);
const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]); const grayscale = tf7.addN([redNorm, greenNorm, blueNorm]);
redNorm.dispose(); tf7.dispose(redNorm);
greenNorm.dispose(); tf7.dispose(greenNorm);
blueNorm.dispose(); tf7.dispose(blueNorm);
const normalize = tf7.tidy(() => grayscale.sub(0.5).mul(2)); const normalize = tf7.tidy(() => tf7.mul(tf7.sub(grayscale, 0.5), 2));
grayscale.dispose(); tf7.dispose(grayscale);
const obj = []; const obj = [];
if (config3.face.emotion.enabled) { if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize); const emotionT = await model2.predict(normalize);
@ -4312,7 +4312,7 @@ async function predict3(image18, config3, idx, count2) {
} }
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); tf7.dispose(normalize);
last2[idx] = obj; last2[idx] = obj;
lastCount2 = count2; lastCount2 = count2;
resolve(obj); resolve(obj);
@ -4636,7 +4636,7 @@ async function predict4(input, config3) {
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]); const resized = tf8.image.resizeBilinear(input, [model3.inputs[0].shape[2], model3.inputs[0].shape[1]]);
const normalized = resized.toFloat().div(127.5).sub(1); const normalized = tf8.sub(tf8.div(tf8.cast(resized, "float32"), 127.5), 1);
const results = model3.execute(normalized, poseNetOutputs); const results = model3.execute(normalized, poseNetOutputs);
const results3d = results.map((y) => tf8.squeeze(y, [0])); const results3d = results.map((y) => tf8.squeeze(y, [0]));
results3d[1] = results3d[1].sigmoid(); results3d[1] = results3d[1].sigmoid();
@ -4644,7 +4644,7 @@ async function predict4(input, config3) {
}); });
const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer())); const buffers = await Promise.all(res.map((tensor2) => tensor2.buffer()));
for (const t of res) for (const t of res)
t.dispose(); tf8.dispose(t);
const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence); const decoded = await decode(buffers[0], buffers[1], buffers[2], buffers[3], config3.body.maxDetected, config3.body.minConfidence);
if (!model3.inputs[0].shape) if (!model3.inputs[0].shape)
return []; return [];
@ -7693,43 +7693,43 @@ var HandDetector = class {
} }
normalizeLandmarks(rawPalmLandmarks, index) { normalizeLandmarks(rawPalmLandmarks, index) {
return tf10.tidy(() => { return tf10.tidy(() => {
const landmarks = tf10.add(tf10.div(rawPalmLandmarks.reshape([-1, 7, 2]), this.inputSizeTensor), this.anchors[index]); const landmarks = tf10.add(tf10.div(tf10.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf10.mul(landmarks, this.inputSizeTensor); return tf10.mul(landmarks, this.inputSizeTensor);
}); });
} }
async getBoxes(input, config3) { async getBoxes(input, config3) {
const batched = this.model.predict(input); const batched = this.model.predict(input);
const predictions = tf10.squeeze(batched); const predictions = tf10.squeeze(batched);
batched.dispose(); tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1])).squeeze()); const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync(); const scores = scoresT.dataSync();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose(); tf10.dispose(rawBoxes);
const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence); const filteredT = await tf10.image.nonMaxSuppressionAsync(boxes, scores, config3.hand.maxDetected, config3.hand.iouThreshold, config3.hand.minConfidence);
const filtered = filteredT.arraySync(); const filtered = filteredT.arraySync();
scoresT.dispose(); tf10.dispose(scoresT);
filteredT.dispose(); tf10.dispose(filteredT);
const hands = []; const hands = [];
for (const index of filtered) { for (const index of filtered) {
if (scores[index] >= config3.hand.minConfidence) { if (scores[index] >= config3.hand.minConfidence) {
const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]); const matchingBox = tf10.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]); const rawPalmLandmarks = tf10.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf10.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, index).reshape([-1, 2])); const palmLandmarks = tf10.tidy(() => tf10.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
rawPalmLandmarks.dispose(); tf10.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] }); hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
} }
} }
predictions.dispose(); tf10.dispose(predictions);
boxes.dispose(); tf10.dispose(boxes);
return hands; return hands;
} }
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image18 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image18 = tf10.tidy(() => tf10.sub(tf10.div(tf10.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image18, config3); const predictions = await this.getBoxes(image18, config3);
image18.dispose(); tf10.dispose(image18);
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -7738,8 +7738,8 @@ var HandDetector = class {
const startPoint = boxes.slice(0, 2); const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4); const endPoint = boxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.box.dispose(); tf10.dispose(prediction.box);
prediction.palmLandmarks.dispose(); tf10.dispose(prediction.palmLandmarks);
hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); hands.push(scaleBoxCoordinates2({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
} }
return hands; return hands;
@ -7902,18 +7902,18 @@ var HandPipeline = class {
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
const handImage = croppedInput.div(255); const handImage = tf11.div(croppedInput, 255);
croppedInput.dispose(); tf11.dispose(croppedInput);
rotatedImage.dispose(); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
handImage.dispose(); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = confidenceT.dataSync()[0];
confidenceT.dispose(); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync(); const rawCoords = keypointsReshaped.arraySync();
keypoints3.dispose(); tf11.dispose(keypoints3);
keypointsReshaped.dispose(); tf11.dispose(keypointsReshaped);
const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords3 = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords3); const nextBoundingBox = this.getBoxForHandLandmarks(coords3);
this.storedBoxes[i] = { ...nextBoundingBox, confidence }; this.storedBoxes[i] = { ...nextBoundingBox, confidence };
@ -7926,7 +7926,7 @@ var HandPipeline = class {
} else { } else {
this.storedBoxes[i] = null; this.storedBoxes[i] = null;
} }
keypoints3.dispose(); tf11.dispose(keypoints3);
} else { } else {
const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor); const enlarged = enlargeBox2(squarifyBox2(currentBox), handBoxEnlargeFactor);
const result = { const result = {
@ -8131,11 +8131,11 @@ async function predict6(image18, config3) {
const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 }; const imgSize = { width: image18.shape[2] || 0, height: image18.shape[1] || 0 };
const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false); const resize = tf13.image.resizeBilinear(image18, [model4["width"], model4["height"]], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); tf13.dispose(resize);
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || []; const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
resT.forEach((t) => t.dispose()); resT.forEach((t) => tf13.dispose(t));
normalize.dispose(); tf13.dispose(normalize);
const keypoints3 = []; const keypoints3 = [];
const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper; const labels2 = (points == null ? void 0 : points.length) === 195 ? full : upper;
const depth = 5; const depth = 5;
@ -8223,7 +8223,7 @@ async function predict7(image18, config3) {
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model5.predict(tensor2); resT = await model5.predict(tensor2);
tensor2.dispose(); tf14.dispose(tensor2);
if (resT) { if (resT) {
keypoints.length = 0; keypoints.length = 0;
const squeeze7 = resT.squeeze(); const squeeze7 = resT.squeeze();
@ -8301,13 +8301,13 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast2 = tf15.cast(resize, "int32"); const cast5 = tf15.cast(resize, "int32");
return cast2; return cast5;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
resT = await model6.predict(tensor2); resT = await model6.predict(tensor2);
tensor2.dispose(); tf15.dispose(tensor2);
if (resT) { if (resT) {
keypoints2.length = 0; keypoints2.length = 0;
const res = resT.arraySync(); const res = resT.arraySync();
@ -8529,14 +8529,14 @@ async function predict9(image18, config3) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image18.shape[2], image18.shape[1]]; const outputSize = [image18.shape[2], image18.shape[1]];
const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false); const resize = tf16.image.resizeBilinear(image18, [model7.inputSize, model7.inputSize], false);
const norm = resize.div(255); const norm = tf16.div(resize, 255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); tf16.dispose(norm);
resize.dispose(); tf16.dispose(resize);
let objectT; let objectT;
if (config3.object.enabled) if (config3.object.enabled)
objectT = await model7.predict(transpose); objectT = await model7.predict(transpose);
transpose.dispose(); tf16.dispose(transpose);
const obj = await process2(objectT, model7.inputSize, outputSize, config3); const obj = await process2(objectT, model7.inputSize, outputSize, config3);
last3 = obj; last3 = obj;
resolve(obj); resolve(obj);
@ -8569,20 +8569,20 @@ async function process3(res, inputSize, outputShape, config3) {
const results = []; const results = [];
const detections = res.arraySync(); const detections = res.arraySync();
const squeezeT = tf17.squeeze(res); const squeezeT = tf17.squeeze(res);
res.dispose(); tf17.dispose(res);
const arr = tf17.split(squeezeT, 6, 1); const arr = tf17.split(squeezeT, 6, 1);
squeezeT.dispose(); tf17.dispose(squeezeT);
const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1); const stackT = tf17.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze(); const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze(); const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze(); const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose()); arr.forEach((t) => tf17.dispose(t));
const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence); const nmsT = await tf17.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose(); tf17.dispose(boxesT);
scoresT.dispose(); tf17.dispose(scoresT);
classesT.dispose(); tf17.dispose(classesT);
const nms = nmsT.dataSync(); const nms = nmsT.dataSync();
nmsT.dispose(); tf17.dispose(nmsT);
let i = 0; let i = 0;
for (const id of nms) { for (const id of nms) {
const score3 = Math.trunc(100 * detections[0][id][4]) / 100; const score3 = Math.trunc(100 * detections[0][id][4]) / 100;
@ -8618,7 +8618,7 @@ async function predict10(input, config3) {
const outputSize = [input.shape[2], input.shape[1]]; const outputSize = [input.shape[2], input.shape[1]];
const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]); const resize = tf17.image.resizeBilinear(input, [model8.inputSize, model8.inputSize]);
const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null; const objectT = config3.object.enabled ? model8.execute(resize, ["tower_0/detections"]) : null;
resize.dispose(); tf17.dispose(resize);
const obj = await process3(objectT, model8.inputSize, outputSize, config3); const obj = await process3(objectT, model8.inputSize, outputSize, config3);
last4 = obj; last4 = obj;
resolve(obj); resolve(obj);
@ -9467,10 +9467,10 @@ function process4(input, config3) {
pixels = tf18.browser ? tf18.browser.fromPixels(data) : null; pixels = tf18.browser ? tf18.browser.fromPixels(data) : null;
} }
if (pixels) { if (pixels) {
const casted = pixels.toFloat(); const casted = tf18.cast(pixels, "float32");
tensor2 = casted.expandDims(0); tensor2 = tf18.expandDims(casted, 0);
pixels.dispose(); tf18.dispose(pixels);
casted.dispose(); tf18.dispose(casted);
} }
} }
const canvas2 = config3.filter.return ? outCanvas : null; const canvas2 = config3.filter.return ? outCanvas : null;
@ -9500,7 +9500,7 @@ async function predict11(input) {
if (!model9 || !model9.inputs[0].shape) if (!model9 || !model9.inputs[0].shape)
return null; return null;
const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false); const resizeInput = tf19.image.resizeBilinear(input.tensor, [model9.inputs[0].shape[1], model9.inputs[0].shape[2]], false);
const norm = resizeInput.div(255); const norm = tf19.div(resizeInput, 255);
const res = model9.predict(norm); const res = model9.predict(norm);
tf19.dispose(resizeInput); tf19.dispose(resizeInput);
tf19.dispose(norm); tf19.dispose(norm);
@ -9509,13 +9509,13 @@ async function predict11(input) {
if (squeeze7.shape[2] === 2) { if (squeeze7.shape[2] === 2) {
const softmax = squeeze7.softmax(); const softmax = squeeze7.softmax();
const [bg, fg] = tf19.unstack(softmax, 2); const [bg, fg] = tf19.unstack(softmax, 2);
const expand = fg.expandDims(2); const expand = tf19.expandDims(fg, 2);
const pad = expand.expandDims(0); const pad = tf19.expandDims(expand, 0);
tf19.dispose(softmax); tf19.dispose(softmax);
tf19.dispose(bg); tf19.dispose(bg);
tf19.dispose(fg); tf19.dispose(fg);
const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]); const crop = tf19.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]);
resizeOutput = crop.squeeze(0); resizeOutput = tf19.squeeze(crop, 0);
tf19.dispose(crop); tf19.dispose(crop);
tf19.dispose(expand); tf19.dispose(expand);
tf19.dispose(pad); tf19.dispose(pad);
@ -11419,7 +11419,7 @@ var Human = class {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return false; return false;
const resizeFact = 32; const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync(); const reducedData = reduced.dataSync();
let sum = 0; let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++) for (let i = 0; i < reducedData.length / 3; i++)
@ -11610,7 +11610,7 @@ var Human = class {
if (elapsedTime > 0) if (elapsedTime > 0)
this.performance.segmentation = elapsedTime; this.performance.segmentation = elapsedTime;
if (process6.canvas) { if (process6.canvas) {
process6.tensor.dispose(); tf21.dispose(process6.tensor);
process6 = process4(process6.canvas, this.config); process6 = process4(process6.canvas, this.config);
} }
this.analyze("End Segmentation:"); this.analyze("End Segmentation:");

4
src/age/age.ts
View File

@ -42,13 +42,13 @@ export async function predict(image: Tensor, config: Config | any) {
const obj = { age: 0 }; const obj = { age: 0 };
if (config.face.age.enabled) ageT = await model.predict(enhance); if (config.face.age.enabled) ageT = await model.predict(enhance);
enhance.dispose(); tf.dispose(enhance);
if (ageT) { if (ageT) {
const data = ageT.dataSync(); const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10; obj.age = Math.trunc(10 * data[0]) / 10;
} }
ageT.dispose(); tf.dispose(ageT);
last = obj; last = obj;
resolve(obj); resolve(obj);

18
src/blazeface/blazeface.ts
View File

@ -43,7 +43,7 @@ export class BlazeFaceModel {
if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null; if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null;
const [batch, boxes, scores] = tf.tidy(() => { const [batch, boxes, scores] = tf.tidy(() => {
const resizedImage = tf.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]); const resizedImage = tf.image.resizeBilinear(inputImage, [this.inputSize, this.inputSize]);
const normalizedImage = resizedImage.div(127.5).sub(0.5); const normalizedImage = tf.sub(tf.div(resizedImage, 127.5), 0.5);
const res = this.model.execute(normalizedImage); const res = this.model.execute(normalizedImage);
let batchOut; let batchOut;
if (Array.isArray(res)) { // are we using tfhub or pinto converted model? if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
@ -51,13 +51,13 @@ export class BlazeFaceModel {
const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16 const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16 const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
const concat = tf.concat([concat512, concat384], 1); const concat = tf.concat([concat512, concat384], 1);
batchOut = concat.squeeze(0); batchOut = tf.squeeze(concat, 0);
} else { } else {
batchOut = tf.squeeze(res); // when using tfhub model batchOut = tf.squeeze(res); // when using tfhub model
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf.slice(batchOut, [0, 0], [-1, 1]); const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf.sigmoid(logits).squeeze().dataSync(); const scoresOut = tf.squeeze(tf.sigmoid(logits)).dataSync();
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
@ -65,23 +65,21 @@ export class BlazeFaceModel {
const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence); const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
const nms = nmsTensor.arraySync(); const nms = nmsTensor.arraySync();
nmsTensor.dispose(); tf.dispose(nmsTensor);
const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: number[], confidence: number }> = []; const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: number[], confidence: number }> = [];
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scores[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = box.createBox(boundingBox); const localBox = box.createBox(boundingBox);
boundingBox.dispose(); tf.dispose(boundingBox);
const anchor = this.anchorsData[nms[i]]; const anchor = this.anchorsData[nms[i]];
const landmarks = tf.tidy(() => tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1])); const landmarks = tf.tidy(() => tf.reshape(tf.squeeze(tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1])), [keypointsCount, -1]));
annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence }); annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
} }
} }
// boundingBoxes.forEach((t) => t.dispose()); tf.dispose(batch);
batch.dispose(); tf.dispose(boxes);
boxes.dispose();
// scores.dispose();
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize], scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize],

4
src/blazeface/box.ts
View File

@ -60,8 +60,8 @@ export function calculateLandmarksBoundingBox(landmarks) {
} }
export const disposeBox = (t) => { export const disposeBox = (t) => {
t.startPoint.dispose(); tf.dispose(t.startPoint);
t.endPoint.dispose(); tf.dispose(t.endPoint);
}; };
export const createBox = (startEndTensor) => ({ export const createBox = (startEndTensor) => ({

2
src/blazeface/facemesh.ts
View File

@ -54,7 +54,7 @@ export async function predict(input: Tensor, config: Config): Promise<Face[]> {
image: prediction.image, image: prediction.image,
tensor: prediction.image, tensor: prediction.image,
}); });
if (prediction.coords) prediction.coords.dispose(); if (prediction.coords) tf.dispose(prediction.coords);
} }
return results; return results;
} }

18
src/blazeface/facepipeline.ts
View File

@ -190,9 +190,9 @@ export class Pipeline {
} }
if (detector && detector.boxes) { if (detector && detector.boxes) {
detector.boxes.forEach((prediction) => { detector.boxes.forEach((prediction) => {
prediction.box.startPoint.dispose(); tf.dispose(prediction.box.startPoint);
prediction.box.endPoint.dispose(); tf.dispose(prediction.box.endPoint);
prediction.landmarks.dispose(); tf.dispose(prediction.landmarks);
}); });
} }
const results = tf.tidy(() => this.storedBoxes.map((box, i) => { const results = tf.tidy(() => this.storedBoxes.map((box, i) => {
@ -208,13 +208,13 @@ export class Pipeline {
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node const rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = util.buildRotationMatrix(-angle, faceCenter); rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
if (config.face.mesh.enabled) face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); if (config.face.mesh.enabled) face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.boxSize, this.boxSize]).div(255); else face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = util.IDENTITY_MATRIX; rotationMatrix = util.IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config.face.mesh.enabled) face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.meshSize, this.meshSize]).div(255); if (config.face.mesh.enabled) face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255);
else face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.boxSize, this.boxSize]).div(255); else face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255);
} }
// if we're not going to produce mesh, don't spend time with further processing // if we're not going to produce mesh, don't spend time with further processing
@ -277,9 +277,9 @@ export class Pipeline {
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]); angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node const rotatedImage = tf.image.rotateWithOffset(tf.cast(input, 'float32'), angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = util.buildRotationMatrix(-angle, faceCenter); rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]).div(255); face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
} }
const prediction = { const prediction = {

6
src/blazepose/blazepose.ts
View File

@ -31,11 +31,11 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
const imgSize = { width: (image.shape[2] || 0), height: (image.shape[1] || 0) }; const imgSize = { width: (image.shape[2] || 0), height: (image.shape[1] || 0) };
const resize = tf.image.resizeBilinear(image, [model['width'], model['height']], false); const resize = tf.image.resizeBilinear(image, [model['width'], model['height']], false);
const normalize = tf.div(resize, [255.0]); const normalize = tf.div(resize, [255.0]);
resize.dispose(); tf.dispose(resize);
const resT = await model.predict(normalize) as Array<Tensor>; const resT = await model.predict(normalize) as Array<Tensor>;
const points = resT.find((t) => (t.size === 195 || t.size === 155))?.dataSync() || []; // order of output tensors may change between models, full has 195 and upper has 155 items const points = resT.find((t) => (t.size === 195 || t.size === 155))?.dataSync() || []; // order of output tensors may change between models, full has 195 and upper has 155 items
resT.forEach((t) => t.dispose()); resT.forEach((t) => tf.dispose(t));
normalize.dispose(); tf.dispose(normalize);
const keypoints: Array<{ id, part, position: [number, number, number], positionRaw: [number, number, number], score, presence }> = []; const keypoints: Array<{ id, part, position: [number, number, number], positionRaw: [number, number, number], score, presence }> = [];
const labels = points?.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints const labels = points?.length === 195 ? annotations.full : annotations.upper; // full model has 39 keypoints, upper has 31 keypoints
const depth = 5; // each points has x,y,z,visibility,presence const depth = 5; // each points has x,y,z,visibility,presence

2
src/efficientpose/efficientpose.ts
View File

@ -68,7 +68,7 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
let resT; let resT;
if (config.body.enabled) resT = await model.predict(tensor); if (config.body.enabled) resT = await model.predict(tensor);
tensor.dispose(); tf.dispose(tensor);
if (resT) { if (resT) {
keypoints.length = 0; keypoints.length = 0;

4
src/embedding/embedding.ts
View File

@ -80,8 +80,8 @@ export function enhance(input): Tensor {
*/ */
// normalize brightness from 0..1 // normalize brightness from 0..1
const darken = merge.sub(merge.min()); const darken = tf.sub(merge, merge.min());
const lighten = darken.div(darken.max()); const lighten = tf.div(darken, darken.max());
return lighten; return lighten;
}); });

20
src/emotion/emotion.ts
View File

@ -36,20 +36,20 @@ export async function predict(image: Tensor, config: Config, idx, count) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf.split(resize, 3, 3); const [red, green, blue] = tf.split(resize, 3, 3);
resize.dispose(); tf.dispose(resize);
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html // weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const redNorm = tf.mul(red, rgb[0]); const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]); const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]); const blueNorm = tf.mul(blue, rgb[2]);
red.dispose(); tf.dispose(red);
green.dispose(); tf.dispose(green);
blue.dispose(); tf.dispose(blue);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]); const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
redNorm.dispose(); tf.dispose(redNorm);
greenNorm.dispose(); tf.dispose(greenNorm);
blueNorm.dispose(); tf.dispose(blueNorm);
const normalize = tf.tidy(() => grayscale.sub(0.5).mul(2)); const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
grayscale.dispose(); tf.dispose(grayscale);
const obj: Array<{ score: number, emotion: string }> = []; const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion.enabled) { if (config.face.emotion.enabled) {
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
@ -60,7 +60,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
} }
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); tf.dispose(normalize);
last[idx] = obj; last[idx] = obj;
lastCount = count; lastCount = count;
resolve(obj); resolve(obj);

4
src/faceres/faceres.ts
View File

@ -104,7 +104,7 @@ export function enhance(input): Tensor {
const lighten = darken.div(darken.max()); const lighten = darken.div(darken.max());
*/ */
const norm = crop.mul(255); const norm = tf.mul(crop, 255);
return norm; return norm;
}); });
@ -140,7 +140,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
obj.genderScore = Math.min(0.99, confidence); obj.genderScore = Math.min(0.99, confidence);
} }
const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0]; const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
const all = resT.find((t) => t.shape[1] === 100).dataSync(); const all = resT.find((t) => t.shape[1] === 100).dataSync();
obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;

6
src/gender/gender.ts
View File

@ -47,7 +47,7 @@ export async function predict(image: Tensor, config: Config | any) {
const greenNorm = tf.mul(green, rgb[1]); const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]); const blueNorm = tf.mul(blue, rgb[2]);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]); const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
const normalize = grayscale.sub(0.5).mul(2); // range grayscale:-1..1 const normalize = tf.mul(tf.sub(grayscale, 0.5), 2); // range grayscale:-1..1
return normalize; return normalize;
}); });
} else { } else {
@ -59,7 +59,7 @@ export async function predict(image: Tensor, config: Config | any) {
const obj = { gender: '', confidence: 0 }; const obj = { gender: '', confidence: 0 };
if (config.face.gender.enabled) genderT = await model.predict(enhance); if (config.face.gender.enabled) genderT = await model.predict(enhance);
enhance.dispose(); tf.dispose(enhance);
if (genderT) { if (genderT) {
if (!Array.isArray(genderT)) { if (!Array.isArray(genderT)) {
@ -78,7 +78,7 @@ export async function predict(image: Tensor, config: Config | any) {
obj.confidence = Math.min(0.99, confidence); obj.confidence = Math.min(0.99, confidence);
} }
} }
genderT.dispose(); tf.dispose(genderT);
} else { } else {
const gender = genderT[0].dataSync(); const gender = genderT[0].dataSync();
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;

28
src/handpose/handdetector.ts
View File

@ -35,7 +35,7 @@ export class HandDetector {
normalizeLandmarks(rawPalmLandmarks, index) { normalizeLandmarks(rawPalmLandmarks, index) {
return tf.tidy(() => { return tf.tidy(() => {
const landmarks = tf.add(tf.div(rawPalmLandmarks.reshape([-1, 7, 2]), this.inputSizeTensor), this.anchors[index]); const landmarks = tf.add(tf.div(tf.reshape(rawPalmLandmarks, [-1, 7, 2]), this.inputSizeTensor), this.anchors[index]);
return tf.mul(landmarks, this.inputSizeTensor); return tf.mul(landmarks, this.inputSizeTensor);
}); });
} }
@ -43,38 +43,38 @@ export class HandDetector {
async getBoxes(input, config) { async getBoxes(input, config) {
const batched = this.model.predict(input) as Tensor; const batched = this.model.predict(input) as Tensor;
const predictions = tf.squeeze(batched); const predictions = tf.squeeze(batched);
batched.dispose(); tf.dispose(batched);
const scoresT = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze()); const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync(); const scores = scoresT.dataSync();
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]); const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
rawBoxes.dispose(); tf.dispose(rawBoxes);
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
const filtered = filteredT.arraySync(); const filtered = filteredT.arraySync();
scoresT.dispose(); tf.dispose(scoresT);
filteredT.dispose(); tf.dispose(filteredT);
const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = []; const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = [];
for (const index of filtered) { for (const index of filtered) {
if (scores[index] >= config.hand.minConfidence) { if (scores[index] >= config.hand.minConfidence) {
const matchingBox = tf.slice(boxes, [index, 0], [1, -1]); const matchingBox = tf.slice(boxes, [index, 0], [1, -1]);
const rawPalmLandmarks = tf.slice(predictions, [index, 5], [1, 14]); const rawPalmLandmarks = tf.slice(predictions, [index, 5], [1, 14]);
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, index).reshape([-1, 2])); const palmLandmarks = tf.tidy(() => tf.reshape(this.normalizeLandmarks(rawPalmLandmarks, index), [-1, 2]));
rawPalmLandmarks.dispose(); tf.dispose(rawPalmLandmarks);
hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] }); hands.push({ box: matchingBox, palmLandmarks, confidence: scores[index] });
} }
} }
predictions.dispose(); tf.dispose(predictions);
boxes.dispose(); tf.dispose(boxes);
return hands; return hands;
} }
async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> { async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image = tf.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image = tf.tidy(() => tf.sub(tf.div(tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]), 127.5), 1));
const predictions = await this.getBoxes(image, config); const predictions = await this.getBoxes(image, config);
image.dispose(); tf.dispose(image);
const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = []; const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
if (!predictions || predictions.length === 0) return hands; if (!predictions || predictions.length === 0) return hands;
for (const prediction of predictions) { for (const prediction of predictions) {
@ -82,8 +82,8 @@ export class HandDetector {
const startPoint = boxes.slice(0, 2); const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4); const endPoint = boxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.box.dispose(); tf.dispose(prediction.box);
prediction.palmLandmarks.dispose(); tf.dispose(prediction.palmLandmarks);
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
} }
return hands; return hands;

16
src/handpose/handpipeline.ts
View File

@ -113,18 +113,18 @@ export class HandPipeline {
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter); const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
const handImage = croppedInput.div(255); const handImage = tf.div(croppedInput, 255);
croppedInput.dispose(); tf.dispose(croppedInput);
rotatedImage.dispose(); tf.dispose(rotatedImage);
const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>; const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>;
handImage.dispose(); tf.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = confidenceT.dataSync()[0];
confidenceT.dispose(); tf.dispose(confidenceT);
if (confidence >= config.hand.minConfidence) { if (confidence >= config.hand.minConfidence) {
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]); const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync(); const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose(); tf.dispose(keypoints);
keypointsReshaped.dispose(); tf.dispose(keypointsReshaped);
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords); const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.storedBoxes[i] = { ...nextBoundingBox, confidence }; this.storedBoxes[i] = { ...nextBoundingBox, confidence };
@ -137,7 +137,7 @@ export class HandPipeline {
} else { } else {
this.storedBoxes[i] = null; this.storedBoxes[i] = null;
} }
keypoints.dispose(); tf.dispose(keypoints);
} else { } else {
// const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), handBoxEnlargeFactor); // const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), handBoxEnlargeFactor);
const enlarged = box.enlargeBox(box.squarifyBox(currentBox), handBoxEnlargeFactor); const enlarged = box.enlargeBox(box.squarifyBox(currentBox), handBoxEnlargeFactor);

4
src/human.ts
View File

@ -357,7 +357,7 @@ export class Human {
#skipFrame = async (input) => { #skipFrame = async (input) => {
if (this.config.cacheSensitivity === 0) return false; if (this.config.cacheSensitivity === 0) return false;
const resizeFact = 32; const resizeFact = 32;
const reduced: Tensor = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced: Tensor = tf.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
// use tensor sum // use tensor sum
/* /*
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
@ -448,7 +448,7 @@ export class Human {
if (elapsedTime > 0) this.performance.segmentation = elapsedTime; if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
if (process.canvas) { if (process.canvas) {
// replace input // replace input
process.tensor.dispose(); tf.dispose(process.tensor);
process = image.process(process.canvas, this.config); process = image.process(process.canvas, this.config);
} }
this.analyze('End Segmentation:'); this.analyze('End Segmentation:');

8
src/image/image.ts
View File

@ -161,10 +161,10 @@ export function process(input: Input, config: Config): { tensor: Tensor | null,
pixels = tf.browser ? tf.browser.fromPixels(data) : null; pixels = tf.browser ? tf.browser.fromPixels(data) : null;
} }
if (pixels) { if (pixels) {
const casted = pixels.toFloat(); const casted = tf.cast(pixels, 'float32');
tensor = casted.expandDims(0); tensor = tf.expandDims(casted, 0);
pixels.dispose(); tf.dispose(pixels);
casted.dispose(); tf.dispose(casted);
} }
} }
const canvas = config.filter.return ? outCanvas : null; const canvas = config.filter.return ? outCanvas : null;

2
src/movenet/movenet.ts
View File

@ -46,7 +46,7 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
let resT; let resT;
if (config.body.enabled) resT = await model.predict(tensor); if (config.body.enabled) resT = await model.predict(tensor);
tensor.dispose(); tf.dispose(tensor);
if (resT) { if (resT) {
keypoints.length = 0; keypoints.length = 0;

16
src/object/centernet.ts
View File

@ -30,20 +30,20 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) {
const results: Array<Item> = []; const results: Array<Item> = [];
const detections = res.arraySync(); const detections = res.arraySync();
const squeezeT = tf.squeeze(res); const squeezeT = tf.squeeze(res);
res.dispose(); tf.dispose(res);
const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class
squeezeT.dispose(); tf.dispose(squeezeT);
const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x
const boxesT = stackT.squeeze(); const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze(); const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze(); const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose()); arr.forEach((t) => tf.dispose(t));
const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
boxesT.dispose(); tf.dispose(boxesT);
scoresT.dispose(); tf.dispose(scoresT);
classesT.dispose(); tf.dispose(classesT);
const nms = nmsT.dataSync(); const nms = nmsT.dataSync();
nmsT.dispose(); tf.dispose(nmsT);
let i = 0; let i = 0;
for (const id of nms) { for (const id of nms) {
const score = Math.trunc(100 * detections[0][id][4]) / 100; const score = Math.trunc(100 * detections[0][id][4]) / 100;
@ -80,7 +80,7 @@ export async function predict(input: Tensor, config: Config): Promise<Item[]> {
const outputSize = [input.shape[2], input.shape[1]]; const outputSize = [input.shape[2], input.shape[1]];
const resize = tf.image.resizeBilinear(input, [model.inputSize, model.inputSize]); const resize = tf.image.resizeBilinear(input, [model.inputSize, model.inputSize]);
const objectT = config.object.enabled ? model.execute(resize, ['tower_0/detections']) : null; const objectT = config.object.enabled ? model.execute(resize, ['tower_0/detections']) : null;
resize.dispose(); tf.dispose(resize);
const obj = await process(objectT, model.inputSize, outputSize, config); const obj = await process(objectT, model.inputSize, outputSize, config);
last = obj; last = obj;

8
src/object/nanodet.ts
View File

@ -111,14 +111,14 @@ export async function predict(image: Tensor, config: Config): Promise<Item[]> {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image.shape[2], image.shape[1]]; const outputSize = [image.shape[2], image.shape[1]];
const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false); const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false);
const norm = resize.div(255); const norm = tf.div(resize, 255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); tf.dispose(norm);
resize.dispose(); tf.dispose(resize);
let objectT; let objectT;
if (config.object.enabled) objectT = await model.predict(transpose); if (config.object.enabled) objectT = await model.predict(transpose);
transpose.dispose(); tf.dispose(transpose);
const obj = await process(objectT, model.inputSize, outputSize, config); const obj = await process(objectT, model.inputSize, outputSize, config);
last = obj; last = obj;

4
src/posenet/posenet.ts
View File

@ -17,7 +17,7 @@ export async function predict(input: Tensor, config: Config): Promise<Body[]> {
const res = tf.tidy(() => { const res = tf.tidy(() => {
if (!model.inputs[0].shape) return []; if (!model.inputs[0].shape) return [];
const resized = tf.image.resizeBilinear(input, [model.inputs[0].shape[2], model.inputs[0].shape[1]]); const resized = tf.image.resizeBilinear(input, [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
const normalized = resized.toFloat().div(127.5).sub(1.0); const normalized = tf.sub(tf.div(tf.cast(resized, 'float32'), 127.5), 1.0);
const results: Array<Tensor> = model.execute(normalized, poseNetOutputs) as Array<Tensor>; const results: Array<Tensor> = model.execute(normalized, poseNetOutputs) as Array<Tensor>;
const results3d = results.map((y) => tf.squeeze(y, [0])); const results3d = results.map((y) => tf.squeeze(y, [0]));
results3d[1] = results3d[1].sigmoid(); // apply sigmoid on scores results3d[1] = results3d[1].sigmoid(); // apply sigmoid on scores
@ -25,7 +25,7 @@ export async function predict(input: Tensor, config: Config): Promise<Body[]> {
}); });
const buffers = await Promise.all(res.map((tensor) => tensor.buffer())); const buffers = await Promise.all(res.map((tensor) => tensor.buffer()));
for (const t of res) t.dispose(); for (const t of res) tf.dispose(t);
const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.maxDetected, config.body.minConfidence); const decoded = await poses.decode(buffers[0], buffers[1], buffers[2], buffers[3], config.body.maxDetected, config.body.minConfidence);
if (!model.inputs[0].shape) return []; if (!model.inputs[0].shape) return [];

8
src/segmentation/segmentation.ts
View File

@ -29,7 +29,7 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
if (!input.tensor) return null; if (!input.tensor) return null;
if (!model || !model.inputs[0].shape) return null; if (!model || !model.inputs[0].shape) return null;
const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false); const resizeInput = tf.image.resizeBilinear(input.tensor, [model.inputs[0].shape[1], model.inputs[0].shape[2]], false);
const norm = resizeInput.div(255); const norm = tf.div(resizeInput, 255);
const res = model.predict(norm) as Tensor; const res = model.predict(norm) as Tensor;
// meet output: 1,256,256,1 // meet output: 1,256,256,1
// selfie output: 1,144,256,2 // selfie output: 1,144,256,2
@ -42,8 +42,8 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
// model meet has two channels for fg and bg // model meet has two channels for fg and bg
const softmax = squeeze.softmax(); const softmax = squeeze.softmax();
const [bg, fg] = tf.unstack(softmax, 2); const [bg, fg] = tf.unstack(softmax, 2);
const expand = fg.expandDims(2); const expand = tf.expandDims(fg, 2);
const pad = expand.expandDims(0); const pad = tf.expandDims(expand, 0);
tf.dispose(softmax); tf.dispose(softmax);
tf.dispose(bg); tf.dispose(bg);
tf.dispose(fg); tf.dispose(fg);
@ -51,7 +51,7 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]); const crop = tf.image.cropAndResize(pad, [[0, 0, 0.5, 0.5]], [0], [width, height]);
// otherwise run softmax after unstack and use standard resize // otherwise run softmax after unstack and use standard resize
// resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]); // resizeOutput = tf.image.resizeBilinear(expand, [input.tensor?.shape[1], input.tensor?.shape[2]]);
resizeOutput = crop.squeeze(0); resizeOutput = tf.squeeze(crop, 0);
tf.dispose(crop); tf.dispose(crop);
tf.dispose(expand); tf.dispose(expand);
tf.dispose(pad); tf.dispose(pad);