mirror of https://github.com/vladmandic/human
blazeface optimizations
Vladimir Mandic
parent
4880048595
commit
5951d0f0d1
2
TODO.md
2
TODO.md
|
|
@ -18,4 +18,4 @@ N/A
|
|||
|
||||
## In Progress
|
||||
|
||||
N/A
|
||||
- Refactor BlazeFace to avoid data and array access and use buffers where needed
|
||||
|
|
|
|||
|
|
@ -6,21 +6,21 @@ import Menu from './helpers/menu.js';
|
|||
import GLBench from './helpers/gl-bench.js';
|
||||
import webRTC from './helpers/webrtc.js';
|
||||
|
||||
const userConfig = { warmup: 'none' };
|
||||
let human;
|
||||
|
||||
/*
|
||||
const userConfig = {
|
||||
warmup: 'none',
|
||||
/*
|
||||
backend: 'humangl',
|
||||
async: false,
|
||||
profile: false,
|
||||
warmup: 'full',
|
||||
videoOptimized: false,
|
||||
videoOptimized: true,
|
||||
filter: {
|
||||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: true,
|
||||
detector: { return: false },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
description: { enabled: false },
|
||||
|
|
@ -31,8 +31,8 @@ const userConfig = {
|
|||
body: { enabled: false, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
// object: { enabled: true },
|
||||
*/
|
||||
};
|
||||
*/
|
||||
|
||||
// ui options
|
||||
const ui = {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -44,22 +44,26 @@ var __privateSet = (obj, member, value, setter) => {
|
|||
|
||||
// dist/tfjs.esm.js
|
||||
var require_tfjs_esm = __commonJS((exports) => {
|
||||
var s = Object.create;
|
||||
var t = Object.defineProperty;
|
||||
var d = Object.getPrototypeOf;
|
||||
var g = Object.prototype.hasOwnProperty;
|
||||
var j = Object.getOwnPropertyNames;
|
||||
var l = Object.getOwnPropertyDescriptor;
|
||||
var p = (o) => t(o, "__esModule", {value: true});
|
||||
var r = (o, e, n) => {
|
||||
if (e && typeof e == "object" || typeof e == "function")
|
||||
for (let f of j(e))
|
||||
!g.call(o, f) && f !== "default" && t(o, f, {get: () => e[f], enumerable: !(n = l(e, f)) || n.enumerable});
|
||||
return o;
|
||||
var __create2 = Object.create;
|
||||
var __defProp2 = Object.defineProperty;
|
||||
var __getProtoOf2 = Object.getPrototypeOf;
|
||||
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
|
||||
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
||||
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
|
||||
var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
|
||||
var __reExport2 = (target, module22, desc) => {
|
||||
if (module22 && typeof module22 === "object" || typeof module22 === "function") {
|
||||
for (let key of __getOwnPropNames2(module22))
|
||||
if (!__hasOwnProp2.call(target, key) && key !== "default")
|
||||
__defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
|
||||
}
|
||||
return target;
|
||||
};
|
||||
var m = (o) => r(p(t(o != null ? s(d(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
|
||||
p(exports);
|
||||
r(exports, m(require("@tensorflow/tfjs-node-gpu")));
|
||||
var __toModule2 = (module22) => {
|
||||
return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
|
||||
};
|
||||
__markAsModule2(exports);
|
||||
__reExport2(exports, __toModule2(require("@tensorflow/tfjs-node-gpu")));
|
||||
});
|
||||
|
||||
// src/human.ts
|
||||
|
|
@ -254,8 +258,8 @@ function squarifyBox(box4) {
|
|||
const size = getBoxSize(box4);
|
||||
const maxEdge = Math.max(...size);
|
||||
const halfSize = maxEdge / 2;
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
|
||||
const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
|
||||
const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
|
||||
return {startPoint, endPoint, landmarks: box4.landmarks};
|
||||
}
|
||||
function calculateLandmarksBoundingBox(landmarks) {
|
||||
|
|
@ -266,7 +270,6 @@ function calculateLandmarksBoundingBox(landmarks) {
|
|||
return {startPoint, endPoint, landmarks};
|
||||
}
|
||||
var createBox = (startEndTensor) => ({
|
||||
startEndTensor,
|
||||
startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
|
||||
endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
|
||||
});
|
||||
|
|
@ -387,46 +390,39 @@ var BlazeFaceModel = class {
|
|||
const [batch, boxes, scores] = tf3.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
|
||||
const normalizedImage = resizedImage.div(127.5).sub(0.5);
|
||||
const batchedPrediction = this.model.predict(normalizedImage);
|
||||
const res = this.model.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(batchedPrediction)) {
|
||||
const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf3.concat([concat512, concat384], 1);
|
||||
batchOut = concat3.squeeze(0);
|
||||
} else {
|
||||
batchOut = batchedPrediction.squeeze();
|
||||
batchOut = res.squeeze();
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
|
||||
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze();
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const boxIndices = boxIndicesTensor.arraySync();
|
||||
boxIndicesTensor.dispose();
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
|
||||
const vals = boundingBox.arraySync();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
});
|
||||
const scoresVal = scores.dataSync();
|
||||
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const nms = nmsTensor.arraySync();
|
||||
nmsTensor.dispose();
|
||||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boxIndex = boxIndices[i];
|
||||
const confidence = scoresVal[boxIndex];
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > this.config.face.detector.minConfidence) {
|
||||
const localBox = createBox(boundingBoxes[i]);
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const localBox = createBox(boundingBox);
|
||||
boundingBox.dispose();
|
||||
const anchor = this.anchorsData[nms[i]];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
|
||||
}
|
||||
}
|
||||
batch.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
|
||||
|
|
@ -3784,9 +3780,9 @@ var Pipeline = class {
|
|||
const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
|
||||
const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
|
||||
return coordsRotated.map((coord) => [
|
||||
coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
|
||||
coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
|
||||
coord[2]
|
||||
Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
|
||||
Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
|
||||
Math.round(coord[2])
|
||||
]);
|
||||
}
|
||||
getLeftToRightEyeDepthDifference(rawCoords) {
|
||||
|
|
@ -3878,8 +3874,7 @@ var Pipeline = class {
|
|||
prediction.landmarks.dispose();
|
||||
});
|
||||
}
|
||||
let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
const boxConfidence = box4.confidence;
|
||||
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
let face4;
|
||||
let angle = 0;
|
||||
let rotationMatrix;
|
||||
|
|
@ -3904,19 +3899,21 @@ var Pipeline = class {
|
|||
}
|
||||
if (!config3.face.mesh.enabled) {
|
||||
const prediction2 = {
|
||||
coords: null,
|
||||
mesh: [],
|
||||
box: box4,
|
||||
faceConfidence: null,
|
||||
boxConfidence,
|
||||
boxConfidence: box4.confidence,
|
||||
confidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
return prediction2;
|
||||
}
|
||||
const [, confidence, contourCoords] = this.meshDetector.predict(face4);
|
||||
const [, confidence, contourCoords] = this.meshDetector.execute(face4);
|
||||
const faceConfidence = confidence.dataSync()[0];
|
||||
if (faceConfidence < config3.face.detector.minConfidence)
|
||||
if (faceConfidence < config3.face.detector.minConfidence) {
|
||||
this.storedBoxes[i].confidence = faceConfidence;
|
||||
return null;
|
||||
}
|
||||
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
|
||||
let rawCoords = coordsReshaped.arraySync();
|
||||
if (config3.face.iris.enabled) {
|
||||
|
|
@ -3941,9 +3938,10 @@ var Pipeline = class {
|
|||
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
|
||||
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
|
||||
}
|
||||
const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
|
||||
const transformedCoords = tf4.tensor2d(transformedCoordsData);
|
||||
const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
const storeConfidence = box4.confidence;
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
|
||||
box4.confidence = storeConfidence;
|
||||
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
|
||||
const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
|
||||
angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
|
||||
|
|
@ -3954,20 +3952,20 @@ var Pipeline = class {
|
|||
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
|
||||
}
|
||||
const prediction = {
|
||||
coords: transformedCoords,
|
||||
mesh,
|
||||
box: box4,
|
||||
faceConfidence,
|
||||
boxConfidence,
|
||||
image: face4,
|
||||
rawCoords
|
||||
boxConfidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
const squarifiedLandmarksBox = squarifyBox(box4);
|
||||
this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
|
||||
const storedBox = squarifyBox(box4);
|
||||
storedBox.confidence = box4.confidence;
|
||||
storedBox.faceConfidence = faceConfidence;
|
||||
this.storedBoxes[i] = storedBox;
|
||||
return prediction;
|
||||
}));
|
||||
results = results.filter((a) => a !== null);
|
||||
if (config3.face.mesh.enabled)
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
|
||||
this.detectedFaces = results.length;
|
||||
return results;
|
||||
}
|
||||
|
|
@ -3980,20 +3978,19 @@ async function predict(input, config3) {
|
|||
const predictions = await facePipeline.predict(input, config3);
|
||||
const results = [];
|
||||
for (const prediction of predictions || []) {
|
||||
if (prediction.isDisposedInternal)
|
||||
if (!prediction || prediction.isDisposedInternal)
|
||||
continue;
|
||||
const mesh = prediction.coords ? prediction.coords.arraySync() : [];
|
||||
const meshRaw = mesh.map((pt) => [
|
||||
const meshRaw = prediction.mesh.map((pt) => [
|
||||
pt[0] / input.shape[2],
|
||||
pt[1] / input.shape[1],
|
||||
pt[2] / facePipeline.meshSize
|
||||
]);
|
||||
const annotations3 = {};
|
||||
if (mesh && mesh.length > 0) {
|
||||
if (prediction.mesh && prediction.mesh.length > 0) {
|
||||
for (const key of Object.keys(MESH_ANNOTATIONS))
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
|
||||
}
|
||||
const box4 = prediction.box ? [
|
||||
const clampedBox = prediction.box ? [
|
||||
Math.max(0, prediction.box.startPoint[0]),
|
||||
Math.max(0, prediction.box.startPoint[1]),
|
||||
Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
|
||||
|
|
@ -4009,17 +4006,15 @@ async function predict(input, config3) {
|
|||
confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
|
||||
boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
|
||||
faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
|
||||
box: box4,
|
||||
box: clampedBox,
|
||||
boxRaw,
|
||||
mesh,
|
||||
mesh: prediction.mesh,
|
||||
meshRaw,
|
||||
annotations: annotations3,
|
||||
image: prediction.image ? prediction.image.clone() : null
|
||||
image: prediction.image
|
||||
});
|
||||
if (prediction.coords)
|
||||
prediction.coords.dispose();
|
||||
if (prediction.image)
|
||||
prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,23 +44,27 @@ var __privateSet = (obj, member, value, setter) => {
|
|||
|
||||
// dist/tfjs.esm.js
|
||||
var require_tfjs_esm = __commonJS((exports) => {
|
||||
var w = Object.create;
|
||||
var e = Object.defineProperty;
|
||||
var a = Object.getPrototypeOf;
|
||||
var j = Object.prototype.hasOwnProperty;
|
||||
var l = Object.getOwnPropertyNames;
|
||||
var p = Object.getOwnPropertyDescriptor;
|
||||
var m = (o) => e(o, "__esModule", {value: true});
|
||||
var f = (o, r, s) => {
|
||||
if (r && typeof r == "object" || typeof r == "function")
|
||||
for (let t of l(r))
|
||||
!j.call(o, t) && t !== "default" && e(o, t, {get: () => r[t], enumerable: !(s = p(r, t)) || s.enumerable});
|
||||
return o;
|
||||
var __create2 = Object.create;
|
||||
var __defProp2 = Object.defineProperty;
|
||||
var __getProtoOf2 = Object.getPrototypeOf;
|
||||
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
|
||||
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
||||
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
|
||||
var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
|
||||
var __reExport2 = (target, module22, desc) => {
|
||||
if (module22 && typeof module22 === "object" || typeof module22 === "function") {
|
||||
for (let key of __getOwnPropNames2(module22))
|
||||
if (!__hasOwnProp2.call(target, key) && key !== "default")
|
||||
__defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
|
||||
}
|
||||
return target;
|
||||
};
|
||||
var n = (o) => f(m(e(o != null ? w(a(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
|
||||
m(exports);
|
||||
f(exports, n(require("@tensorflow/tfjs")));
|
||||
f(exports, n(require("@tensorflow/tfjs-backend-wasm")));
|
||||
var __toModule2 = (module22) => {
|
||||
return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
|
||||
};
|
||||
__markAsModule2(exports);
|
||||
__reExport2(exports, __toModule2(require("@tensorflow/tfjs")));
|
||||
__reExport2(exports, __toModule2(require("@tensorflow/tfjs-backend-wasm")));
|
||||
});
|
||||
|
||||
// src/human.ts
|
||||
|
|
@ -255,8 +259,8 @@ function squarifyBox(box4) {
|
|||
const size = getBoxSize(box4);
|
||||
const maxEdge = Math.max(...size);
|
||||
const halfSize = maxEdge / 2;
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
|
||||
const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
|
||||
const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
|
||||
return {startPoint, endPoint, landmarks: box4.landmarks};
|
||||
}
|
||||
function calculateLandmarksBoundingBox(landmarks) {
|
||||
|
|
@ -267,7 +271,6 @@ function calculateLandmarksBoundingBox(landmarks) {
|
|||
return {startPoint, endPoint, landmarks};
|
||||
}
|
||||
var createBox = (startEndTensor) => ({
|
||||
startEndTensor,
|
||||
startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
|
||||
endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
|
||||
});
|
||||
|
|
@ -388,46 +391,39 @@ var BlazeFaceModel = class {
|
|||
const [batch, boxes, scores] = tf3.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
|
||||
const normalizedImage = resizedImage.div(127.5).sub(0.5);
|
||||
const batchedPrediction = this.model.predict(normalizedImage);
|
||||
const res = this.model.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(batchedPrediction)) {
|
||||
const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf3.concat([concat512, concat384], 1);
|
||||
batchOut = concat3.squeeze(0);
|
||||
} else {
|
||||
batchOut = batchedPrediction.squeeze();
|
||||
batchOut = res.squeeze();
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
|
||||
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze();
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const boxIndices = boxIndicesTensor.arraySync();
|
||||
boxIndicesTensor.dispose();
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
|
||||
const vals = boundingBox.arraySync();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
});
|
||||
const scoresVal = scores.dataSync();
|
||||
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const nms = nmsTensor.arraySync();
|
||||
nmsTensor.dispose();
|
||||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boxIndex = boxIndices[i];
|
||||
const confidence = scoresVal[boxIndex];
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > this.config.face.detector.minConfidence) {
|
||||
const localBox = createBox(boundingBoxes[i]);
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const localBox = createBox(boundingBox);
|
||||
boundingBox.dispose();
|
||||
const anchor = this.anchorsData[nms[i]];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
|
||||
}
|
||||
}
|
||||
batch.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
|
||||
|
|
@ -3785,9 +3781,9 @@ var Pipeline = class {
|
|||
const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
|
||||
const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
|
||||
return coordsRotated.map((coord) => [
|
||||
coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
|
||||
coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
|
||||
coord[2]
|
||||
Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
|
||||
Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
|
||||
Math.round(coord[2])
|
||||
]);
|
||||
}
|
||||
getLeftToRightEyeDepthDifference(rawCoords) {
|
||||
|
|
@ -3879,8 +3875,7 @@ var Pipeline = class {
|
|||
prediction.landmarks.dispose();
|
||||
});
|
||||
}
|
||||
let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
const boxConfidence = box4.confidence;
|
||||
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
let face4;
|
||||
let angle = 0;
|
||||
let rotationMatrix;
|
||||
|
|
@ -3905,19 +3900,21 @@ var Pipeline = class {
|
|||
}
|
||||
if (!config3.face.mesh.enabled) {
|
||||
const prediction2 = {
|
||||
coords: null,
|
||||
mesh: [],
|
||||
box: box4,
|
||||
faceConfidence: null,
|
||||
boxConfidence,
|
||||
boxConfidence: box4.confidence,
|
||||
confidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
return prediction2;
|
||||
}
|
||||
const [, confidence, contourCoords] = this.meshDetector.predict(face4);
|
||||
const [, confidence, contourCoords] = this.meshDetector.execute(face4);
|
||||
const faceConfidence = confidence.dataSync()[0];
|
||||
if (faceConfidence < config3.face.detector.minConfidence)
|
||||
if (faceConfidence < config3.face.detector.minConfidence) {
|
||||
this.storedBoxes[i].confidence = faceConfidence;
|
||||
return null;
|
||||
}
|
||||
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
|
||||
let rawCoords = coordsReshaped.arraySync();
|
||||
if (config3.face.iris.enabled) {
|
||||
|
|
@ -3942,9 +3939,10 @@ var Pipeline = class {
|
|||
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
|
||||
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
|
||||
}
|
||||
const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
|
||||
const transformedCoords = tf4.tensor2d(transformedCoordsData);
|
||||
const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
const storeConfidence = box4.confidence;
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
|
||||
box4.confidence = storeConfidence;
|
||||
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
|
||||
const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
|
||||
angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
|
||||
|
|
@ -3955,20 +3953,20 @@ var Pipeline = class {
|
|||
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
|
||||
}
|
||||
const prediction = {
|
||||
coords: transformedCoords,
|
||||
mesh,
|
||||
box: box4,
|
||||
faceConfidence,
|
||||
boxConfidence,
|
||||
image: face4,
|
||||
rawCoords
|
||||
boxConfidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
const squarifiedLandmarksBox = squarifyBox(box4);
|
||||
this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
|
||||
const storedBox = squarifyBox(box4);
|
||||
storedBox.confidence = box4.confidence;
|
||||
storedBox.faceConfidence = faceConfidence;
|
||||
this.storedBoxes[i] = storedBox;
|
||||
return prediction;
|
||||
}));
|
||||
results = results.filter((a) => a !== null);
|
||||
if (config3.face.mesh.enabled)
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
|
||||
this.detectedFaces = results.length;
|
||||
return results;
|
||||
}
|
||||
|
|
@ -3981,20 +3979,19 @@ async function predict(input, config3) {
|
|||
const predictions = await facePipeline.predict(input, config3);
|
||||
const results = [];
|
||||
for (const prediction of predictions || []) {
|
||||
if (prediction.isDisposedInternal)
|
||||
if (!prediction || prediction.isDisposedInternal)
|
||||
continue;
|
||||
const mesh = prediction.coords ? prediction.coords.arraySync() : [];
|
||||
const meshRaw = mesh.map((pt) => [
|
||||
const meshRaw = prediction.mesh.map((pt) => [
|
||||
pt[0] / input.shape[2],
|
||||
pt[1] / input.shape[1],
|
||||
pt[2] / facePipeline.meshSize
|
||||
]);
|
||||
const annotations3 = {};
|
||||
if (mesh && mesh.length > 0) {
|
||||
if (prediction.mesh && prediction.mesh.length > 0) {
|
||||
for (const key of Object.keys(MESH_ANNOTATIONS))
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
|
||||
}
|
||||
const box4 = prediction.box ? [
|
||||
const clampedBox = prediction.box ? [
|
||||
Math.max(0, prediction.box.startPoint[0]),
|
||||
Math.max(0, prediction.box.startPoint[1]),
|
||||
Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
|
||||
|
|
@ -4010,17 +4007,15 @@ async function predict(input, config3) {
|
|||
confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
|
||||
boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
|
||||
faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
|
||||
box: box4,
|
||||
box: clampedBox,
|
||||
boxRaw,
|
||||
mesh,
|
||||
mesh: prediction.mesh,
|
||||
meshRaw,
|
||||
annotations: annotations3,
|
||||
image: prediction.image ? prediction.image.clone() : null
|
||||
image: prediction.image
|
||||
});
|
||||
if (prediction.coords)
|
||||
prediction.coords.dispose();
|
||||
if (prediction.image)
|
||||
prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,22 +44,26 @@ var __privateSet = (obj, member, value, setter) => {
|
|||
|
||||
// dist/tfjs.esm.js
|
||||
var require_tfjs_esm = __commonJS((exports) => {
|
||||
var d = Object.create;
|
||||
var t = Object.defineProperty;
|
||||
var j = Object.getPrototypeOf;
|
||||
var l = Object.prototype.hasOwnProperty;
|
||||
var m = Object.getOwnPropertyNames;
|
||||
var p = Object.getOwnPropertyDescriptor;
|
||||
var s = (o) => t(o, "__esModule", {value: true});
|
||||
var r = (o, e, n) => {
|
||||
if (e && typeof e == "object" || typeof e == "function")
|
||||
for (let f of m(e))
|
||||
!l.call(o, f) && f !== "default" && t(o, f, {get: () => e[f], enumerable: !(n = p(e, f)) || n.enumerable});
|
||||
return o;
|
||||
var __create2 = Object.create;
|
||||
var __defProp2 = Object.defineProperty;
|
||||
var __getProtoOf2 = Object.getPrototypeOf;
|
||||
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
|
||||
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
||||
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
|
||||
var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
|
||||
var __reExport2 = (target, module22, desc) => {
|
||||
if (module22 && typeof module22 === "object" || typeof module22 === "function") {
|
||||
for (let key of __getOwnPropNames2(module22))
|
||||
if (!__hasOwnProp2.call(target, key) && key !== "default")
|
||||
__defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
|
||||
}
|
||||
return target;
|
||||
};
|
||||
var w = (o) => r(s(t(o != null ? d(j(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
|
||||
s(exports);
|
||||
r(exports, w(require("@tensorflow/tfjs-node")));
|
||||
var __toModule2 = (module22) => {
|
||||
return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
|
||||
};
|
||||
__markAsModule2(exports);
|
||||
__reExport2(exports, __toModule2(require("@tensorflow/tfjs-node")));
|
||||
});
|
||||
|
||||
// src/human.ts
|
||||
|
|
@ -254,8 +258,8 @@ function squarifyBox(box4) {
|
|||
const size = getBoxSize(box4);
|
||||
const maxEdge = Math.max(...size);
|
||||
const halfSize = maxEdge / 2;
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
|
||||
const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
|
||||
const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
|
||||
return {startPoint, endPoint, landmarks: box4.landmarks};
|
||||
}
|
||||
function calculateLandmarksBoundingBox(landmarks) {
|
||||
|
|
@ -266,7 +270,6 @@ function calculateLandmarksBoundingBox(landmarks) {
|
|||
return {startPoint, endPoint, landmarks};
|
||||
}
|
||||
var createBox = (startEndTensor) => ({
|
||||
startEndTensor,
|
||||
startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
|
||||
endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
|
||||
});
|
||||
|
|
@ -387,46 +390,39 @@ var BlazeFaceModel = class {
|
|||
const [batch, boxes, scores] = tf3.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
|
||||
const normalizedImage = resizedImage.div(127.5).sub(0.5);
|
||||
const batchedPrediction = this.model.predict(normalizedImage);
|
||||
const res = this.model.execute(normalizedImage);
|
||||
let batchOut;
|
||||
if (Array.isArray(batchedPrediction)) {
|
||||
const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
|
||||
if (Array.isArray(res)) {
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
|
||||
const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
|
||||
const concat3 = tf3.concat([concat512, concat384], 1);
|
||||
batchOut = concat3.squeeze(0);
|
||||
} else {
|
||||
batchOut = batchedPrediction.squeeze();
|
||||
batchOut = res.squeeze();
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
|
||||
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze();
|
||||
const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const boxIndices = boxIndicesTensor.arraySync();
|
||||
boxIndicesTensor.dispose();
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
|
||||
const vals = boundingBox.arraySync();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
});
|
||||
const scoresVal = scores.dataSync();
|
||||
const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const nms = nmsTensor.arraySync();
|
||||
nmsTensor.dispose();
|
||||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boxIndex = boxIndices[i];
|
||||
const confidence = scoresVal[boxIndex];
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > this.config.face.detector.minConfidence) {
|
||||
const localBox = createBox(boundingBoxes[i]);
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const localBox = createBox(boundingBox);
|
||||
boundingBox.dispose();
|
||||
const anchor = this.anchorsData[nms[i]];
|
||||
const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
|
||||
}
|
||||
}
|
||||
batch.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
|
||||
|
|
@ -3784,9 +3780,9 @@ var Pipeline = class {
|
|||
const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
|
||||
const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
|
||||
return coordsRotated.map((coord) => [
|
||||
coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
|
||||
coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
|
||||
coord[2]
|
||||
Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
|
||||
Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
|
||||
Math.round(coord[2])
|
||||
]);
|
||||
}
|
||||
getLeftToRightEyeDepthDifference(rawCoords) {
|
||||
|
|
@ -3878,8 +3874,7 @@ var Pipeline = class {
|
|||
prediction.landmarks.dispose();
|
||||
});
|
||||
}
|
||||
let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
const boxConfidence = box4.confidence;
|
||||
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
|
||||
let face4;
|
||||
let angle = 0;
|
||||
let rotationMatrix;
|
||||
|
|
@ -3904,19 +3899,21 @@ var Pipeline = class {
|
|||
}
|
||||
if (!config3.face.mesh.enabled) {
|
||||
const prediction2 = {
|
||||
coords: null,
|
||||
mesh: [],
|
||||
box: box4,
|
||||
faceConfidence: null,
|
||||
boxConfidence,
|
||||
boxConfidence: box4.confidence,
|
||||
confidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
return prediction2;
|
||||
}
|
||||
const [, confidence, contourCoords] = this.meshDetector.predict(face4);
|
||||
const [, confidence, contourCoords] = this.meshDetector.execute(face4);
|
||||
const faceConfidence = confidence.dataSync()[0];
|
||||
if (faceConfidence < config3.face.detector.minConfidence)
|
||||
if (faceConfidence < config3.face.detector.minConfidence) {
|
||||
this.storedBoxes[i].confidence = faceConfidence;
|
||||
return null;
|
||||
}
|
||||
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
|
||||
let rawCoords = coordsReshaped.arraySync();
|
||||
if (config3.face.iris.enabled) {
|
||||
|
|
@ -3941,9 +3938,10 @@ var Pipeline = class {
|
|||
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
|
||||
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
|
||||
}
|
||||
const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
|
||||
const transformedCoords = tf4.tensor2d(transformedCoordsData);
|
||||
const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
|
||||
const storeConfidence = box4.confidence;
|
||||
box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
|
||||
box4.confidence = storeConfidence;
|
||||
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
|
||||
const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
|
||||
angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
|
||||
|
|
@ -3954,20 +3952,20 @@ var Pipeline = class {
|
|||
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
|
||||
}
|
||||
const prediction = {
|
||||
coords: transformedCoords,
|
||||
mesh,
|
||||
box: box4,
|
||||
faceConfidence,
|
||||
boxConfidence,
|
||||
image: face4,
|
||||
rawCoords
|
||||
boxConfidence: box4.confidence,
|
||||
image: face4
|
||||
};
|
||||
const squarifiedLandmarksBox = squarifyBox(box4);
|
||||
this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
|
||||
const storedBox = squarifyBox(box4);
|
||||
storedBox.confidence = box4.confidence;
|
||||
storedBox.faceConfidence = faceConfidence;
|
||||
this.storedBoxes[i] = storedBox;
|
||||
return prediction;
|
||||
}));
|
||||
results = results.filter((a) => a !== null);
|
||||
if (config3.face.mesh.enabled)
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
|
||||
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
|
||||
this.detectedFaces = results.length;
|
||||
return results;
|
||||
}
|
||||
|
|
@ -3980,20 +3978,19 @@ async function predict(input, config3) {
|
|||
const predictions = await facePipeline.predict(input, config3);
|
||||
const results = [];
|
||||
for (const prediction of predictions || []) {
|
||||
if (prediction.isDisposedInternal)
|
||||
if (!prediction || prediction.isDisposedInternal)
|
||||
continue;
|
||||
const mesh = prediction.coords ? prediction.coords.arraySync() : [];
|
||||
const meshRaw = mesh.map((pt) => [
|
||||
const meshRaw = prediction.mesh.map((pt) => [
|
||||
pt[0] / input.shape[2],
|
||||
pt[1] / input.shape[1],
|
||||
pt[2] / facePipeline.meshSize
|
||||
]);
|
||||
const annotations3 = {};
|
||||
if (mesh && mesh.length > 0) {
|
||||
if (prediction.mesh && prediction.mesh.length > 0) {
|
||||
for (const key of Object.keys(MESH_ANNOTATIONS))
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
|
||||
annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
|
||||
}
|
||||
const box4 = prediction.box ? [
|
||||
const clampedBox = prediction.box ? [
|
||||
Math.max(0, prediction.box.startPoint[0]),
|
||||
Math.max(0, prediction.box.startPoint[1]),
|
||||
Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
|
||||
|
|
@ -4009,17 +4006,15 @@ async function predict(input, config3) {
|
|||
confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
|
||||
boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
|
||||
faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
|
||||
box: box4,
|
||||
box: clampedBox,
|
||||
boxRaw,
|
||||
mesh,
|
||||
mesh: prediction.mesh,
|
||||
meshRaw,
|
||||
annotations: annotations3,
|
||||
image: prediction.image ? prediction.image.clone() : null
|
||||
image: prediction.image
|
||||
});
|
||||
if (prediction.coords)
|
||||
prediction.coords.dispose();
|
||||
if (prediction.image)
|
||||
prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -40,50 +40,42 @@ export class BlazeFaceModel {
|
|||
if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null;
|
||||
const [batch, boxes, scores] = tf.tidy(() => {
|
||||
const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
|
||||
// const normalizedImage = tf.mul(tf.sub(resizedImage.div(255), 0.5), 2);
|
||||
const normalizedImage = resizedImage.div(127.5).sub(0.5);
|
||||
const batchedPrediction = this.model.predict(normalizedImage);
|
||||
const res = this.model.execute(normalizedImage);
|
||||
let batchOut;
|
||||
// are we using tfhub or pinto converted model?
|
||||
if (Array.isArray(batchedPrediction)) {
|
||||
const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
|
||||
if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
|
||||
const sorted = res.sort((a, b) => a.size - b.size);
|
||||
const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
|
||||
const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
|
||||
const concat = tf.concat([concat512, concat384], 1);
|
||||
batchOut = concat.squeeze(0);
|
||||
} else {
|
||||
batchOut = batchedPrediction.squeeze(); // when using tfhub model
|
||||
batchOut = res.squeeze(); // when using tfhub model
|
||||
}
|
||||
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
|
||||
const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
|
||||
const scoresOut = tf.sigmoid(logits).squeeze();
|
||||
const scoresOut = tf.sigmoid(logits).squeeze().dataSync();
|
||||
return [batchOut, boxesOut, scoresOut];
|
||||
});
|
||||
const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const boxIndices = boxIndicesTensor.arraySync();
|
||||
boxIndicesTensor.dispose();
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
|
||||
const vals = boundingBox.arraySync();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
});
|
||||
|
||||
const scoresVal = scores.dataSync();
|
||||
const annotatedBoxes: Array<{ box: any, landmarks: any, anchor: any, confidence: number }> = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boxIndex = boxIndices[i];
|
||||
const confidence = scoresVal[boxIndex];
|
||||
const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
|
||||
const nms = nmsTensor.arraySync();
|
||||
nmsTensor.dispose();
|
||||
const annotatedBoxes: Array<{ box: any, landmarks: any, anchor: number[], confidence: number }> = [];
|
||||
for (let i = 0; i < nms.length; i++) {
|
||||
const confidence = scores[nms[i]];
|
||||
if (confidence > this.config.face.detector.minConfidence) {
|
||||
const localBox = box.createBox(boundingBoxes[i]);
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf.tidy(() => tf.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
|
||||
const localBox = box.createBox(boundingBox);
|
||||
boundingBox.dispose();
|
||||
const anchor = this.anchorsData[nms[i]];
|
||||
const landmarks = tf.tidy(() => tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
|
||||
annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
|
||||
}
|
||||
}
|
||||
// boundingBoxes.forEach((t) => t.dispose());
|
||||
batch.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
// scores.dispose();
|
||||
return {
|
||||
boxes: annotatedBoxes,
|
||||
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize],
|
||||
|
|
|
|||
|
|
@ -46,8 +46,8 @@ export function squarifyBox(box) {
|
|||
const size = getBoxSize(box);
|
||||
const maxEdge = Math.max(...size);
|
||||
const halfSize = maxEdge / 2;
|
||||
const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
|
||||
const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
|
||||
const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
|
||||
const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
|
||||
return { startPoint, endPoint, landmarks: box.landmarks };
|
||||
}
|
||||
|
||||
|
|
@ -60,13 +60,11 @@ export function calculateLandmarksBoundingBox(landmarks) {
|
|||
}
|
||||
|
||||
export const disposeBox = (t) => {
|
||||
t.startEndTensor.dispose();
|
||||
t.startPoint.dispose();
|
||||
t.endPoint.dispose();
|
||||
};
|
||||
|
||||
export const createBox = (startEndTensor) => ({
|
||||
startEndTensor,
|
||||
startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]),
|
||||
endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,18 +11,17 @@ export async function predict(input, config): Promise<{ confidence, boxConfidenc
|
|||
const predictions = await facePipeline.predict(input, config);
|
||||
const results: Array<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }> = [];
|
||||
for (const prediction of (predictions || [])) {
|
||||
if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
|
||||
const mesh = prediction.coords ? prediction.coords.arraySync() : [];
|
||||
const meshRaw = mesh.map((pt) => [
|
||||
if (!prediction || prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
|
||||
const meshRaw = prediction.mesh.map((pt) => [
|
||||
pt[0] / input.shape[2],
|
||||
pt[1] / input.shape[1],
|
||||
pt[2] / facePipeline.meshSize,
|
||||
]);
|
||||
const annotations = {};
|
||||
if (mesh && mesh.length > 0) {
|
||||
for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
|
||||
if (prediction.mesh && prediction.mesh.length > 0) {
|
||||
for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
|
||||
}
|
||||
const box = prediction.box ? [
|
||||
const clampedBox = prediction.box ? [
|
||||
Math.max(0, prediction.box.startPoint[0]),
|
||||
Math.max(0, prediction.box.startPoint[1]),
|
||||
Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
|
||||
|
|
@ -38,15 +37,14 @@ export async function predict(input, config): Promise<{ confidence, boxConfidenc
|
|||
confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
|
||||
boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
|
||||
faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
|
||||
box,
|
||||
box: clampedBox,
|
||||
boxRaw,
|
||||
mesh,
|
||||
mesh: prediction.mesh,
|
||||
meshRaw,
|
||||
annotations,
|
||||
image: prediction.image ? prediction.image.clone() : null,
|
||||
image: prediction.image,
|
||||
});
|
||||
if (prediction.coords) prediction.coords.dispose();
|
||||
if (prediction.image) prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,9 +90,9 @@ export class Pipeline {
|
|||
const inverseRotationMatrix = (angle !== 0) ? util.invertTransformMatrix(rotationMatrix) : util.IDENTITY_MATRIX;
|
||||
const boxCenter = [...bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1];
|
||||
return coordsRotated.map((coord) => ([
|
||||
coord[0] + util.dot(boxCenter, inverseRotationMatrix[0]),
|
||||
coord[1] + util.dot(boxCenter, inverseRotationMatrix[1]),
|
||||
coord[2],
|
||||
Math.round(coord[0] + util.dot(boxCenter, inverseRotationMatrix[0])),
|
||||
Math.round(coord[1] + util.dot(boxCenter, inverseRotationMatrix[1])),
|
||||
Math.round(coord[2]),
|
||||
]));
|
||||
}
|
||||
|
||||
|
|
@ -195,10 +195,7 @@ export class Pipeline {
|
|||
prediction.landmarks.dispose();
|
||||
});
|
||||
}
|
||||
|
||||
let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
|
||||
const boxConfidence = box.confidence;
|
||||
|
||||
const results = tf.tidy(() => this.storedBoxes.map((box, i) => {
|
||||
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
|
||||
let face;
|
||||
let angle = 0;
|
||||
|
|
@ -223,19 +220,22 @@ export class Pipeline {
|
|||
// if we're not going to produce mesh, don't spend time with further processing
|
||||
if (!config.face.mesh.enabled) {
|
||||
const prediction = {
|
||||
coords: null,
|
||||
mesh: [],
|
||||
box,
|
||||
faceConfidence: null,
|
||||
boxConfidence,
|
||||
boxConfidence: box.confidence,
|
||||
confidence: box.confidence,
|
||||
image: face,
|
||||
};
|
||||
return prediction;
|
||||
}
|
||||
|
||||
const [, confidence, contourCoords] = this.meshDetector.predict(face); // The first returned tensor represents facial contours which are already included in the coordinates.
|
||||
const [, confidence, contourCoords] = this.meshDetector.execute(face); // The first returned tensor represents facial contours which are already included in the coordinates.
|
||||
const faceConfidence = confidence.dataSync()[0];
|
||||
if (faceConfidence < config.face.detector.minConfidence) return null; // if below confidence just exit
|
||||
if (faceConfidence < config.face.detector.minConfidence) {
|
||||
this.storedBoxes[i].confidence = faceConfidence; // reset confidence of cached box
|
||||
return null; // if below confidence just exit
|
||||
}
|
||||
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
|
||||
let rawCoords = coordsReshaped.arraySync();
|
||||
|
||||
|
|
@ -265,9 +265,10 @@ export class Pipeline {
|
|||
}
|
||||
|
||||
// override box from detection with one calculated from mesh
|
||||
const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
|
||||
box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(transformedCoordsData), 1.5); // redefine box with mesh calculated one
|
||||
const transformedCoords = tf.tensor2d(transformedCoordsData);
|
||||
const mesh = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
|
||||
const storeConfidence = box.confidence;
|
||||
box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(mesh), 1.5); // redefine box with mesh calculated one
|
||||
box.confidence = storeConfidence;
|
||||
|
||||
// do rotation one more time with mesh keypoints if we want to return perfect image
|
||||
if (config.face.detector.rotation && config.face.mesh.enabled && config.face.description.enabled && tf.ENV.flags.IS_BROWSER) {
|
||||
|
|
@ -281,24 +282,28 @@ export class Pipeline {
|
|||
}
|
||||
|
||||
const prediction = {
|
||||
coords: transformedCoords,
|
||||
mesh,
|
||||
box,
|
||||
faceConfidence,
|
||||
boxConfidence,
|
||||
boxConfidence: box.confidence,
|
||||
image: face,
|
||||
rawCoords,
|
||||
};
|
||||
|
||||
// updated stored cache values
|
||||
const squarifiedLandmarksBox = bounding.squarifyBox(box);
|
||||
this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence };
|
||||
const storedBox = bounding.squarifyBox(box);
|
||||
// @ts-ignore box itself doesn't have those properties, but we stored them for future use
|
||||
storedBox.confidence = box.confidence;
|
||||
// @ts-ignore box itself doesn't have those properties, but we stored them for future use
|
||||
storedBox.faceConfidence = faceConfidence;
|
||||
// this.storedBoxes[i] = { ...squarifiedLandmarksBox, confidence: box.confidence, faceConfidence };
|
||||
this.storedBoxes[i] = storedBox;
|
||||
|
||||
return prediction;
|
||||
}));
|
||||
|
||||
results = results.filter((a) => a !== null);
|
||||
// results = results.filter((a) => a !== null);
|
||||
// remove cache entries for detected boxes on low confidence
|
||||
if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config.face.detector.minConfidence);
|
||||
if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config.face.detector.minConfidence);
|
||||
this.detectedFaces = results.length;
|
||||
|
||||
return results;
|
||||
|
|
|
|||
Loading…
Reference in New Issue