ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

human 1.9.0 beta with breaking changes regarding caching

pull/134/head
Vladimir Mandic 2021-05-18 11:26:16 -04:00
parent b3b43a0e78
commit a62fc0b7d4
31 changed files with 226184 additions and 4908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
demo/face3d.js
View File

@ -9,7 +9,6 @@ const userConfig = {
async: false, async: false,
profile: false, profile: false,
warmup: 'full', warmup: 'full',
videoOptimized: true,
filter: { enabled: false }, filter: { enabled: false },
face: { enabled: true, face: { enabled: true,
detector: { rotation: false, maxDetected: 1 }, detector: { rotation: false, maxDetected: 1 },

3
demo/facematch.js
View File

@ -7,7 +7,6 @@ const userConfig = {
async: false, async: false,
warmup: 'none', warmup: 'none',
debug: true, debug: true,
videoOptimized: false,
face: { face: {
enabled: true, enabled: true,
detector: { rotation: true, return: true }, detector: { rotation: true, return: true },
@ -16,7 +15,7 @@ const userConfig = {
iris: { enabled: false }, iris: { enabled: false },
age: { enabled: false }, age: { enabled: false },
gender: { enabled: false }, gender: { enabled: false },
emotion: { enabled: false }, emotion: { enabled: true },
description: { enabled: true }, description: { enabled: true },
}, },
hand: { enabled: false }, hand: { enabled: false },

2
demo/index.js
View File

@ -13,7 +13,6 @@ const userConfig = {
/* /*
backend: 'webgl', backend: 'webgl',
async: true, async: true,
videoOptimized: false,
filter: { filter: {
enabled: false, enabled: false,
flip: false, flip: false,
@ -487,7 +486,6 @@ async function detectVideo() {
// just initialize everything and call main function // just initialize everything and call main function
async function detectSampleImages() { async function detectSampleImages() {
userConfig.videoOptimized = false; // force disable video optimizations
document.getElementById('canvas').style.display = 'none'; document.getElementById('canvas').style.display = 'none';
document.getElementById('samples-container').style.display = 'block'; document.getElementById('samples-container').style.display = 'block';
log('running detection of sample images'); log('running detection of sample images');

1
demo/node-multiprocess-worker.js
View File

@ -14,7 +14,6 @@ const myConfig = {
backend: 'tensorflow', backend: 'tensorflow',
modelBasePath: 'file://models/', modelBasePath: 'file://models/',
debug: false, debug: false,
videoOptimized: false,
async: true, async: true,
face: { face: {
enabled: true, enabled: true,

1
demo/node-video.js
View File

@ -25,7 +25,6 @@ const humanConfig = {
backend: 'tensorflow', backend: 'tensorflow',
modelBasePath: 'file://node_modules/@vladmandic/human/models/', modelBasePath: 'file://node_modules/@vladmandic/human/models/',
debug: false, debug: false,
videoOptimized: true,
async: true, async: true,
filter: { enabled: false }, filter: { enabled: false },
face: { face: {

1
demo/node.js
View File

@ -16,7 +16,6 @@ const myConfig = {
backend: 'tensorflow', backend: 'tensorflow',
modelBasePath: 'file://models/', modelBasePath: 'file://models/',
debug: true, debug: true,
videoOptimized: false,
async: false, async: false,
filter: { filter: {
enabled: true, enabled: true,

19213
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

75838
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

75844
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

219
dist/human.node-gpu.js vendored
View File

@ -126,8 +126,8 @@ var config = {
wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//", wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//",
debug: true, debug: true,
async: true, async: true,
videoOptimized: true,
warmup: "full", warmup: "full",
cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -158,7 +158,6 @@ var config = {
rotation: false, rotation: false,
maxDetected: 10, maxDetected: 10,
skipFrames: 21, skipFrames: 21,
skipInitial: false,
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.1, iouThreshold: 0.1,
return: false return: false
@ -194,7 +193,6 @@ var config = {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 12,
skipInitial: false,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -340,16 +338,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image11, cropSize) { function cutBoxFromImageAndResize(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image11, boxes, [0], cropSize); return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -3896,10 +3894,10 @@ var Pipeline = class {
const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2]; const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2];
return leftEyeZ - rightEyeZ; return leftEyeZ - rightEyeZ;
} }
getEyeBox(rawCoords, face4, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) { getEyeBox(rawCoords, face5, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) {
const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge)); const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge));
const boxSize = getBoxSize(box4); const boxSize = getBoxSize(box4);
let crop = tf4.image.cropAndResize(face4, [[ let crop = tf4.image.cropAndResize(face5, [[
box4.startPoint[1] / this.meshSize, box4.startPoint[1] / this.meshSize,
box4.startPoint[0] / this.meshSize, box4.startPoint[0] / this.meshSize,
box4.endPoint[1] / this.meshSize, box4.endPoint[1] / this.meshSize,
@ -3941,13 +3939,13 @@ var Pipeline = class {
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.skipFrame) {
detector = await this.boundingBoxDetector.getBoundingBoxes(input); detector = await this.boundingBoxDetector.getBoundingBoxes(input);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (!config3.videoOptimized || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) { if (!config3.skipFrame || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) {
this.storedBoxes = []; this.storedBoxes = [];
this.detectedFaces = 0; this.detectedFaces = 0;
for (const possible of detector.boxes) { for (const possible of detector.boxes) {
@ -3956,8 +3954,6 @@ var Pipeline = class {
if (this.storedBoxes.length > 0) if (this.storedBoxes.length > 0)
useFreshBox = true; useFreshBox = true;
} }
if (config3.face.detector.skipInitial && this.detectedFaces === 0)
this.skipped = 0;
if (useFreshBox) { if (useFreshBox) {
if (!detector || !detector.boxes || detector.boxes.length === 0) { if (!detector || !detector.boxes || detector.boxes.length === 0) {
this.storedBoxes = []; this.storedBoxes = [];
@ -3981,7 +3977,7 @@ var Pipeline = class {
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
let face4; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
@ -3992,16 +3988,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4010,11 +4006,11 @@ var Pipeline = class {
faceConfidence: null, faceConfidence: null,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
confidence: box4.confidence, confidence: box4.confidence,
image: face4 image: face5
}; };
return prediction2; return prediction2;
} }
const [, confidence, contourCoords] = this.meshDetector.execute(face4); const [, confidence, contourCoords] = this.meshDetector.execute(face5);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = confidence.dataSync()[0];
if (faceConfidence < config3.face.detector.minConfidence) { if (faceConfidence < config3.face.detector.minConfidence) {
this.storedBoxes[i].confidence = faceConfidence; this.storedBoxes[i].confidence = faceConfidence;
@ -4023,8 +4019,8 @@ var Pipeline = class {
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
let rawCoords = coordsReshaped.arraySync(); let rawCoords = coordsReshaped.arraySync();
if (config3.face.iris.enabled) { if (config3.face.iris.enabled) {
const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]); const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop])); const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync(); const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3); const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
@ -4055,14 +4051,14 @@ var Pipeline = class {
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
} }
const prediction = { const prediction = {
mesh, mesh,
box: box4, box: box4,
faceConfidence, faceConfidence,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
image: face4 image: face5
}; };
const storedBox = squarifyBox(box4); const storedBox = squarifyBox(box4);
storedBox.confidence = box4.confidence; storedBox.confidence = box4.confidence;
@ -4164,6 +4160,7 @@ var tf6 = __toModule(require_tfjs_esm());
var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]; var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"];
var model; var model;
var last = []; var last = [];
var lastCount = 0;
var skipped = Number.MAX_SAFE_INTEGER; var skipped = Number.MAX_SAFE_INTEGER;
var rgb = [0.2989, 0.587, 0.114]; var rgb = [0.2989, 0.587, 0.114];
async function load3(config3) { async function load3(config3) {
@ -4177,19 +4174,16 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image11, config3) { async function predict2(image12, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.videoOptimized && last.length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
skipped++; skipped++;
return last; return last[idx];
} }
if (config3.videoOptimized)
skipped = 0; skipped = 0;
else
skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image11, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4216,7 +4210,8 @@ async function predict2(image11, config3) {
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); normalize.dispose();
last = obj; last[idx] = obj;
lastCount = count2;
resolve(obj); resolve(obj);
}); });
} }
@ -4232,7 +4227,8 @@ __export(faceres_exports, {
}); });
var tf7 = __toModule(require_tfjs_esm()); var tf7 = __toModule(require_tfjs_esm());
var model2; var model2;
var last2 = {age: 0}; var last2 = [];
var lastCount2 = 0;
var skipped2 = Number.MAX_SAFE_INTEGER; var skipped2 = Number.MAX_SAFE_INTEGER;
async function load4(config3) { async function load4(config3) {
if (!model2) { if (!model2) {
@ -4270,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image11 = tf7.tidy(() => { const image12 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4279,21 +4275,19 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image11; return image12;
} }
async function predict3(image11, config3) { async function predict3(image12, config3, idx, count2) {
var _a, _b;
if (!model2) if (!model2)
return null; return null;
if (skipped2 < config3.face.description.skipFrames && config3.videoOptimized && last2.age && last2.age > 0) { if (skipped2 < config3.face.description.skipFrames && config3.skipFrame && lastCount2 === count2 && ((_a = last2[idx]) == null ? void 0 : _a.age) && ((_b = last2[idx]) == null ? void 0 : _b.age) > 0) {
skipped2++; skipped2++;
return last2; return last2;
} }
if (config3.videoOptimized)
skipped2 = 0; skipped2 = 0;
else
skipped2 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image11); const enhanced = enhance(image12);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4320,13 +4314,14 @@ async function predict3(image11, config3) {
}); });
resT.forEach((t) => tf7.dispose(t)); resT.forEach((t) => tf7.dispose(t));
} }
last2 = obj; last2[idx] = obj;
lastCount2 = count2;
resolve(obj); resolve(obj);
}); });
} }
// src/faceall.ts // src/face.ts
var calculateFaceAngle = (face4, image_size) => { var calculateFaceAngle = (face5, image_size) => {
const degrees = (theta) => theta * 180 / Math.PI; const degrees = (theta) => theta * 180 / Math.PI;
const normalize = (v) => { const normalize = (v) => {
const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
@ -4378,10 +4373,10 @@ var calculateFaceAngle = (face4, image_size) => {
}; };
return angle2; return angle2;
}; };
const mesh = face4.meshRaw; const mesh = face5.meshRaw;
if (!mesh || mesh.length < 300) if (!mesh || mesh.length < 300)
return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]}; return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]};
const size = Math.max(face4.boxRaw[2] * image_size[0], face4.boxRaw[3] * image_size[1]) / 1.5; const size = Math.max(face5.boxRaw[2] * image_size[0], face5.boxRaw[3] * image_size[1]) / 1.5;
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
pt[0] * image_size[0] / size, pt[0] * image_size[0] / size,
pt[1] * image_size[1] / size, pt[1] * image_size[1] / size,
@ -4406,7 +4401,7 @@ var calculateFaceAngle = (face4, image_size) => {
return {angle, matrix}; return {angle, matrix};
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {
var _a, _b, _c, _d, _e, _f; var _a, _b, _c, _d, _e, _f, _g, _h;
let timeStamp; let timeStamp;
let ageRes; let ageRes;
let genderRes; let genderRes;
@ -4420,30 +4415,30 @@ var detectFace = async (parent, input) => {
parent.perf.face = Math.trunc(now() - timeStamp); parent.perf.face = Math.trunc(now() - timeStamp);
if (!faces) if (!faces)
return []; return [];
for (const face4 of faces) { for (let i = 0; i < faces.length; i++) {
parent.analyze("Get Face"); parent.analyze("Get Face");
if (!face4.image || face4.image.isDisposedInternal) { if (!faces[i].image || faces[i].image.isDisposedInternal) {
log("Face object is disposed:", face4.image); log("Face object is disposed:", faces[i].image);
continue; continue;
} }
const rotation = calculateFaceAngle(face4, [input.shape[2], input.shape[1]]); const rotation = calculateFaceAngle(faces[i], [input.shape[2], input.shape[1]]);
parent.analyze("Start Emotion:"); parent.analyze("Start Emotion:");
if (parent.config.async) { if (parent.config.async) {
emotionRes = parent.config.face.emotion.enabled ? predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? predict2(faces[i].image, parent.config, i, faces.length) : {};
} else { } else {
parent.state = "run:emotion"; parent.state = "run:emotion";
timeStamp = now(); timeStamp = now();
emotionRes = parent.config.face.emotion.enabled ? await predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? await predict2(faces[i].image, parent.config, i, faces.length) : {};
parent.perf.emotion = Math.trunc(now() - timeStamp); parent.perf.emotion = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Emotion:"); parent.analyze("End Emotion:");
parent.analyze("Start Description:"); parent.analyze("Start Description:");
if (parent.config.async) { if (parent.config.async) {
descRes = parent.config.face.description.enabled ? predict3(face4, parent.config) : []; descRes = parent.config.face.description.enabled ? predict3(faces[i], parent.config, i, faces.length) : [];
} else { } else {
parent.state = "run:description"; parent.state = "run:description";
timeStamp = now(); timeStamp = now();
descRes = parent.config.face.description.enabled ? await predict3(face4.image, parent.config) : []; descRes = parent.config.face.description.enabled ? await predict3(faces[i].image, parent.config, i, faces.length) : [];
parent.perf.embedding = Math.trunc(now() - timeStamp); parent.perf.embedding = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Description:"); parent.analyze("End Description:");
@ -4451,13 +4446,13 @@ var detectFace = async (parent, input) => {
[ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]); [ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]);
} }
parent.analyze("Finish Face:"); parent.analyze("Finish Face:");
if (!parent.config.face.iris.enabled && ((_a = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _a.leftEyeIris) && ((_b = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _b.rightEyeIris)) { if (!parent.config.face.iris.enabled && ((_b = (_a = faces[i]) == null ? void 0 : _a.annotations) == null ? void 0 : _b.leftEyeIris) && ((_d = (_c = faces[i]) == null ? void 0 : _c.annotations) == null ? void 0 : _d.rightEyeIris)) {
delete face4.annotations.leftEyeIris; delete faces[i].annotations.leftEyeIris;
delete face4.annotations.rightEyeIris; delete faces[i].annotations.rightEyeIris;
} }
const irisSize = ((_c = face4.annotations) == null ? void 0 : _c.leftEyeIris) && ((_d = face4.annotations) == null ? void 0 : _d.rightEyeIris) ? 11.7 * Math.max(Math.abs(face4.annotations.leftEyeIris[3][0] - face4.annotations.leftEyeIris[1][0]), Math.abs(face4.annotations.rightEyeIris[4][1] - face4.annotations.rightEyeIris[2][1])) : 0; const irisSize = ((_e = faces[i].annotations) == null ? void 0 : _e.leftEyeIris) && ((_f = faces[i].annotations) == null ? void 0 : _f.rightEyeIris) ? 11.7 * Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) : 0;
faceRes.push({ faceRes.push({
...face4, ...faces[i],
age: descRes.age, age: descRes.age,
gender: descRes.gender, gender: descRes.gender,
genderConfidence: descRes.genderConfidence, genderConfidence: descRes.genderConfidence,
@ -4465,9 +4460,9 @@ var detectFace = async (parent, input) => {
emotion: emotionRes, emotion: emotionRes,
iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0, iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0,
rotation, rotation,
tensor: parent.config.face.detector.return ? (_e = face4.image) == null ? void 0 : _e.squeeze() : null tensor: parent.config.face.detector.return ? (_g = faces[i].image) == null ? void 0 : _g.squeeze() : null
}); });
(_f = face4.image) == null ? void 0 : _f.dispose(); (_h = faces[i].image) == null ? void 0 : _h.dispose();
parent.analyze("End Face"); parent.analyze("End Face");
} }
parent.analyze("End FaceMesh:"); parent.analyze("End FaceMesh:");
@ -4849,16 +4844,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image11, cropSize) { function cutBoxFromImageAndResize2(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image11, boxes, [0], cropSize); return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16725,9 +16720,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image11 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image11, config3); const predictions = await this.getBoxes(image12, config3);
image11.dispose(); image12.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16872,14 +16867,14 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image11, config3) { async estimateHands(image12, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image11, config3); boxes = await this.handDetector.estimateHandBounds(image12, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) { if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) {
this.detectedHands = 0; this.detectedHands = 0;
@ -16888,8 +16883,6 @@ var HandPipeline = class {
useFreshBox = true; useFreshBox = true;
} }
const hands = []; const hands = [];
if (config3.hand.skipInitial && this.detectedHands === 0)
this.skipped = 0;
for (let i = 0; i < this.storedBoxes.length; i++) { for (let i = 0; i < this.storedBoxes.length; i++) {
const currentBox = this.storedBoxes[i]; const currentBox = this.storedBoxes[i];
if (!currentBox) if (!currentBox)
@ -16897,8 +16890,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image11.shape[2], palmCenter[1] / image11.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image11, angle, 0, palmCenterNormalized) : image11.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17108,13 +17101,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image11, config3) { async function predict6(image12, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image11.shape[2], height: image11.shape[1]}; const imgSize = {width: image12.shape[2], height: image12.shape[1]};
const resize = tf13.image.resizeBilinear(image11, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17317,20 +17310,17 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image11, config3) { async function predict7(image12, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.videoOptimized && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
skipped3++; skipped3++;
return last3; return last3;
} }
if (config3.videoOptimized)
skipped3 = 0; skipped3 = 0;
else
skipped3 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image11.shape[2], image11.shape[1]]; const outputSize = [image12.shape[2], image12.shape[1]];
const resize = tf14.image.resizeBilinear(image11, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17454,10 +17444,6 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var image_exports = {};
__export(image_exports, {
process: () => process3
});
var tf15 = __toModule(require_tfjs_esm()); var tf15 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
@ -17610,8 +17596,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image11) { this.apply = function(image12) {
_resize(image11.width, image11.height); _resize(image12.width, image12.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17620,7 +17606,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image11); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -19525,16 +19511,17 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`; 2Q==`;
// package.json // package.json
var version = "1.8.5"; var version = "1.9.0";
// src/human.ts // src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _sanity, _checkBackend, _warmupBitmap, _warmupCanvas, _warmupNode; var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
var Human = class { var Human = class {
constructor(userConfig = {}) { constructor(userConfig = {}) {
__privateAdd(this, _numTensors, void 0); __privateAdd(this, _numTensors, void 0);
__privateAdd(this, _analyzeMemoryLeaks, void 0); __privateAdd(this, _analyzeMemoryLeaks, void 0);
__privateAdd(this, _checkSanity, void 0); __privateAdd(this, _checkSanity, void 0);
__privateAdd(this, _firstRun, void 0); __privateAdd(this, _firstRun, void 0);
__privateAdd(this, _lastInputSum, void 0);
this.analyze = (...msg) => { this.analyze = (...msg) => {
if (!__privateGet(this, _analyzeMemoryLeaks)) if (!__privateGet(this, _analyzeMemoryLeaks))
return; return;
@ -19611,6 +19598,19 @@ var Human = class {
this.perf.backend = Math.trunc(now() - timeStamp); this.perf.backend = Math.trunc(now() - timeStamp);
} }
}); });
__privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0)
return true;
const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced);
reduced.dispose();
const sum = sumT.dataSync()[0];
sumT.dispose();
const diff = Math.max(sum, __privateGet(this, _lastInputSum)) / Math.min(sum, __privateGet(this, _lastInputSum)) - 1;
__privateSet(this, _lastInputSum, sum);
return diff < this.config.cacheSensitivity;
});
__privateAdd(this, _warmupBitmap, async () => { __privateAdd(this, _warmupBitmap, async () => {
const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob()); const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob());
let blob; let blob;
@ -19721,6 +19721,7 @@ var Human = class {
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
this.sysinfo = info(); this.sysinfo = info();
__privateSet(this, _lastInputSum, 1);
} }
similarity(embedding1, embedding2) { similarity(embedding1, embedding2) {
return similarity(embedding1, embedding2); return similarity(embedding1, embedding2);
@ -19810,12 +19811,6 @@ var Human = class {
const timeStart = now(); const timeStart = now();
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
let previousVideoOptimized;
if (input && this.config.videoOptimized && typeof window !== "undefined" && typeof WorkerGlobalScope !== "undefined" && (typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement || typeof Image !== "undefined" && input instanceof Image || typeof ImageData !== "undefined" && input instanceof ImageData || typeof ImageBitmap !== "undefined" && image_exports instanceof ImageBitmap)) {
log("disabling video optimization");
previousVideoOptimized = this.config.videoOptimized;
this.config.videoOptimized = false;
}
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process4 = process3(input, this.config);
if (!process4 || !process4.tensor) { if (!process4 || !process4.tensor) {
@ -19825,6 +19820,17 @@ var Human = class {
} }
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
if (!this.perf.frames)
this.perf.frames = 0;
if (!this.perf.cached)
this.perf.cached = 0;
this.perf.frames++;
if (this.config.skipFrame)
this.perf.cached++;
this.perf.changed = Math.trunc(now() - timeStamp);
this.analyze("Check Changed:");
let bodyRes; let bodyRes;
let handRes; let handRes;
let faceRes; let faceRes;
@ -19903,8 +19909,6 @@ var Human = class {
else if (this.perf.gesture) else if (this.perf.gesture)
delete this.perf.gesture; delete this.perf.gesture;
} }
if (previousVideoOptimized)
this.config.videoOptimized = previousVideoOptimized;
this.perf.total = Math.trunc(now() - timeStart); this.perf.total = Math.trunc(now() - timeStart);
this.state = "idle"; this.state = "idle";
const result = { const result = {
@ -19925,8 +19929,6 @@ var Human = class {
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
if (!this.config.warmup || this.config.warmup === "none") if (!this.config.warmup || this.config.warmup === "none")
return {error: "null"}; return {error: "null"};
const save = this.config.videoOptimized;
this.config.videoOptimized = false;
let res; let res;
if (typeof createImageBitmap === "function") if (typeof createImageBitmap === "function")
res = await __privateGet(this, _warmupBitmap).call(this); res = await __privateGet(this, _warmupBitmap).call(this);
@ -19934,7 +19936,6 @@ var Human = class {
res = await __privateGet(this, _warmupCanvas).call(this); res = await __privateGet(this, _warmupCanvas).call(this);
else else
res = await __privateGet(this, _warmupNode).call(this); res = await __privateGet(this, _warmupNode).call(this);
this.config.videoOptimized = save;
const t1 = now(); const t1 = now();
if (this.config.debug) if (this.config.debug)
log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res); log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res);
@ -19945,8 +19946,10 @@ _numTensors = new WeakMap();
_analyzeMemoryLeaks = new WeakMap(); _analyzeMemoryLeaks = new WeakMap();
_checkSanity = new WeakMap(); _checkSanity = new WeakMap();
_firstRun = new WeakMap(); _firstRun = new WeakMap();
_lastInputSum = new WeakMap();
_sanity = new WeakMap(); _sanity = new WeakMap();
_checkBackend = new WeakMap(); _checkBackend = new WeakMap();
_skipFrame = new WeakMap();
_warmupBitmap = new WeakMap(); _warmupBitmap = new WeakMap();
_warmupCanvas = new WeakMap(); _warmupCanvas = new WeakMap();
_warmupNode = new WeakMap(); _warmupNode = new WeakMap();

219
dist/human.node-wasm.js vendored
View File

@ -127,8 +127,8 @@ var config = {
wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//", wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//",
debug: true, debug: true,
async: true, async: true,
videoOptimized: true,
warmup: "full", warmup: "full",
cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -159,7 +159,6 @@ var config = {
rotation: false, rotation: false,
maxDetected: 10, maxDetected: 10,
skipFrames: 21, skipFrames: 21,
skipInitial: false,
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.1, iouThreshold: 0.1,
return: false return: false
@ -195,7 +194,6 @@ var config = {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 12,
skipInitial: false,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -341,16 +339,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image11, cropSize) { function cutBoxFromImageAndResize(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image11, boxes, [0], cropSize); return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -3897,10 +3895,10 @@ var Pipeline = class {
const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2]; const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2];
return leftEyeZ - rightEyeZ; return leftEyeZ - rightEyeZ;
} }
getEyeBox(rawCoords, face4, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) { getEyeBox(rawCoords, face5, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) {
const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge)); const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge));
const boxSize = getBoxSize(box4); const boxSize = getBoxSize(box4);
let crop = tf4.image.cropAndResize(face4, [[ let crop = tf4.image.cropAndResize(face5, [[
box4.startPoint[1] / this.meshSize, box4.startPoint[1] / this.meshSize,
box4.startPoint[0] / this.meshSize, box4.startPoint[0] / this.meshSize,
box4.endPoint[1] / this.meshSize, box4.endPoint[1] / this.meshSize,
@ -3942,13 +3940,13 @@ var Pipeline = class {
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.skipFrame) {
detector = await this.boundingBoxDetector.getBoundingBoxes(input); detector = await this.boundingBoxDetector.getBoundingBoxes(input);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (!config3.videoOptimized || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) { if (!config3.skipFrame || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) {
this.storedBoxes = []; this.storedBoxes = [];
this.detectedFaces = 0; this.detectedFaces = 0;
for (const possible of detector.boxes) { for (const possible of detector.boxes) {
@ -3957,8 +3955,6 @@ var Pipeline = class {
if (this.storedBoxes.length > 0) if (this.storedBoxes.length > 0)
useFreshBox = true; useFreshBox = true;
} }
if (config3.face.detector.skipInitial && this.detectedFaces === 0)
this.skipped = 0;
if (useFreshBox) { if (useFreshBox) {
if (!detector || !detector.boxes || detector.boxes.length === 0) { if (!detector || !detector.boxes || detector.boxes.length === 0) {
this.storedBoxes = []; this.storedBoxes = [];
@ -3982,7 +3978,7 @@ var Pipeline = class {
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
let face4; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
@ -3993,16 +3989,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4011,11 +4007,11 @@ var Pipeline = class {
faceConfidence: null, faceConfidence: null,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
confidence: box4.confidence, confidence: box4.confidence,
image: face4 image: face5
}; };
return prediction2; return prediction2;
} }
const [, confidence, contourCoords] = this.meshDetector.execute(face4); const [, confidence, contourCoords] = this.meshDetector.execute(face5);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = confidence.dataSync()[0];
if (faceConfidence < config3.face.detector.minConfidence) { if (faceConfidence < config3.face.detector.minConfidence) {
this.storedBoxes[i].confidence = faceConfidence; this.storedBoxes[i].confidence = faceConfidence;
@ -4024,8 +4020,8 @@ var Pipeline = class {
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
let rawCoords = coordsReshaped.arraySync(); let rawCoords = coordsReshaped.arraySync();
if (config3.face.iris.enabled) { if (config3.face.iris.enabled) {
const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]); const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop])); const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync(); const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3); const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
@ -4056,14 +4052,14 @@ var Pipeline = class {
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
} }
const prediction = { const prediction = {
mesh, mesh,
box: box4, box: box4,
faceConfidence, faceConfidence,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
image: face4 image: face5
}; };
const storedBox = squarifyBox(box4); const storedBox = squarifyBox(box4);
storedBox.confidence = box4.confidence; storedBox.confidence = box4.confidence;
@ -4165,6 +4161,7 @@ var tf6 = __toModule(require_tfjs_esm());
var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]; var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"];
var model; var model;
var last = []; var last = [];
var lastCount = 0;
var skipped = Number.MAX_SAFE_INTEGER; var skipped = Number.MAX_SAFE_INTEGER;
var rgb = [0.2989, 0.587, 0.114]; var rgb = [0.2989, 0.587, 0.114];
async function load3(config3) { async function load3(config3) {
@ -4178,19 +4175,16 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image11, config3) { async function predict2(image12, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.videoOptimized && last.length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
skipped++; skipped++;
return last; return last[idx];
} }
if (config3.videoOptimized)
skipped = 0; skipped = 0;
else
skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image11, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4217,7 +4211,8 @@ async function predict2(image11, config3) {
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); normalize.dispose();
last = obj; last[idx] = obj;
lastCount = count2;
resolve(obj); resolve(obj);
}); });
} }
@ -4233,7 +4228,8 @@ __export(faceres_exports, {
}); });
var tf7 = __toModule(require_tfjs_esm()); var tf7 = __toModule(require_tfjs_esm());
var model2; var model2;
var last2 = {age: 0}; var last2 = [];
var lastCount2 = 0;
var skipped2 = Number.MAX_SAFE_INTEGER; var skipped2 = Number.MAX_SAFE_INTEGER;
async function load4(config3) { async function load4(config3) {
if (!model2) { if (!model2) {
@ -4271,7 +4267,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image11 = tf7.tidy(() => { const image12 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4280,21 +4276,19 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image11; return image12;
} }
async function predict3(image11, config3) { async function predict3(image12, config3, idx, count2) {
var _a, _b;
if (!model2) if (!model2)
return null; return null;
if (skipped2 < config3.face.description.skipFrames && config3.videoOptimized && last2.age && last2.age > 0) { if (skipped2 < config3.face.description.skipFrames && config3.skipFrame && lastCount2 === count2 && ((_a = last2[idx]) == null ? void 0 : _a.age) && ((_b = last2[idx]) == null ? void 0 : _b.age) > 0) {
skipped2++; skipped2++;
return last2; return last2;
} }
if (config3.videoOptimized)
skipped2 = 0; skipped2 = 0;
else
skipped2 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image11); const enhanced = enhance(image12);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4321,13 +4315,14 @@ async function predict3(image11, config3) {
}); });
resT.forEach((t) => tf7.dispose(t)); resT.forEach((t) => tf7.dispose(t));
} }
last2 = obj; last2[idx] = obj;
lastCount2 = count2;
resolve(obj); resolve(obj);
}); });
} }
// src/faceall.ts // src/face.ts
var calculateFaceAngle = (face4, image_size) => { var calculateFaceAngle = (face5, image_size) => {
const degrees = (theta) => theta * 180 / Math.PI; const degrees = (theta) => theta * 180 / Math.PI;
const normalize = (v) => { const normalize = (v) => {
const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
@ -4379,10 +4374,10 @@ var calculateFaceAngle = (face4, image_size) => {
}; };
return angle2; return angle2;
}; };
const mesh = face4.meshRaw; const mesh = face5.meshRaw;
if (!mesh || mesh.length < 300) if (!mesh || mesh.length < 300)
return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]}; return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]};
const size = Math.max(face4.boxRaw[2] * image_size[0], face4.boxRaw[3] * image_size[1]) / 1.5; const size = Math.max(face5.boxRaw[2] * image_size[0], face5.boxRaw[3] * image_size[1]) / 1.5;
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
pt[0] * image_size[0] / size, pt[0] * image_size[0] / size,
pt[1] * image_size[1] / size, pt[1] * image_size[1] / size,
@ -4407,7 +4402,7 @@ var calculateFaceAngle = (face4, image_size) => {
return {angle, matrix}; return {angle, matrix};
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {
var _a, _b, _c, _d, _e, _f; var _a, _b, _c, _d, _e, _f, _g, _h;
let timeStamp; let timeStamp;
let ageRes; let ageRes;
let genderRes; let genderRes;
@ -4421,30 +4416,30 @@ var detectFace = async (parent, input) => {
parent.perf.face = Math.trunc(now() - timeStamp); parent.perf.face = Math.trunc(now() - timeStamp);
if (!faces) if (!faces)
return []; return [];
for (const face4 of faces) { for (let i = 0; i < faces.length; i++) {
parent.analyze("Get Face"); parent.analyze("Get Face");
if (!face4.image || face4.image.isDisposedInternal) { if (!faces[i].image || faces[i].image.isDisposedInternal) {
log("Face object is disposed:", face4.image); log("Face object is disposed:", faces[i].image);
continue; continue;
} }
const rotation = calculateFaceAngle(face4, [input.shape[2], input.shape[1]]); const rotation = calculateFaceAngle(faces[i], [input.shape[2], input.shape[1]]);
parent.analyze("Start Emotion:"); parent.analyze("Start Emotion:");
if (parent.config.async) { if (parent.config.async) {
emotionRes = parent.config.face.emotion.enabled ? predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? predict2(faces[i].image, parent.config, i, faces.length) : {};
} else { } else {
parent.state = "run:emotion"; parent.state = "run:emotion";
timeStamp = now(); timeStamp = now();
emotionRes = parent.config.face.emotion.enabled ? await predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? await predict2(faces[i].image, parent.config, i, faces.length) : {};
parent.perf.emotion = Math.trunc(now() - timeStamp); parent.perf.emotion = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Emotion:"); parent.analyze("End Emotion:");
parent.analyze("Start Description:"); parent.analyze("Start Description:");
if (parent.config.async) { if (parent.config.async) {
descRes = parent.config.face.description.enabled ? predict3(face4, parent.config) : []; descRes = parent.config.face.description.enabled ? predict3(faces[i], parent.config, i, faces.length) : [];
} else { } else {
parent.state = "run:description"; parent.state = "run:description";
timeStamp = now(); timeStamp = now();
descRes = parent.config.face.description.enabled ? await predict3(face4.image, parent.config) : []; descRes = parent.config.face.description.enabled ? await predict3(faces[i].image, parent.config, i, faces.length) : [];
parent.perf.embedding = Math.trunc(now() - timeStamp); parent.perf.embedding = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Description:"); parent.analyze("End Description:");
@ -4452,13 +4447,13 @@ var detectFace = async (parent, input) => {
[ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]); [ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]);
} }
parent.analyze("Finish Face:"); parent.analyze("Finish Face:");
if (!parent.config.face.iris.enabled && ((_a = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _a.leftEyeIris) && ((_b = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _b.rightEyeIris)) { if (!parent.config.face.iris.enabled && ((_b = (_a = faces[i]) == null ? void 0 : _a.annotations) == null ? void 0 : _b.leftEyeIris) && ((_d = (_c = faces[i]) == null ? void 0 : _c.annotations) == null ? void 0 : _d.rightEyeIris)) {
delete face4.annotations.leftEyeIris; delete faces[i].annotations.leftEyeIris;
delete face4.annotations.rightEyeIris; delete faces[i].annotations.rightEyeIris;
} }
const irisSize = ((_c = face4.annotations) == null ? void 0 : _c.leftEyeIris) && ((_d = face4.annotations) == null ? void 0 : _d.rightEyeIris) ? 11.7 * Math.max(Math.abs(face4.annotations.leftEyeIris[3][0] - face4.annotations.leftEyeIris[1][0]), Math.abs(face4.annotations.rightEyeIris[4][1] - face4.annotations.rightEyeIris[2][1])) : 0; const irisSize = ((_e = faces[i].annotations) == null ? void 0 : _e.leftEyeIris) && ((_f = faces[i].annotations) == null ? void 0 : _f.rightEyeIris) ? 11.7 * Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) : 0;
faceRes.push({ faceRes.push({
...face4, ...faces[i],
age: descRes.age, age: descRes.age,
gender: descRes.gender, gender: descRes.gender,
genderConfidence: descRes.genderConfidence, genderConfidence: descRes.genderConfidence,
@ -4466,9 +4461,9 @@ var detectFace = async (parent, input) => {
emotion: emotionRes, emotion: emotionRes,
iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0, iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0,
rotation, rotation,
tensor: parent.config.face.detector.return ? (_e = face4.image) == null ? void 0 : _e.squeeze() : null tensor: parent.config.face.detector.return ? (_g = faces[i].image) == null ? void 0 : _g.squeeze() : null
}); });
(_f = face4.image) == null ? void 0 : _f.dispose(); (_h = faces[i].image) == null ? void 0 : _h.dispose();
parent.analyze("End Face"); parent.analyze("End Face");
} }
parent.analyze("End FaceMesh:"); parent.analyze("End FaceMesh:");
@ -4850,16 +4845,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image11, cropSize) { function cutBoxFromImageAndResize2(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image11, boxes, [0], cropSize); return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16726,9 +16721,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image11 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image11, config3); const predictions = await this.getBoxes(image12, config3);
image11.dispose(); image12.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16873,14 +16868,14 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image11, config3) { async estimateHands(image12, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image11, config3); boxes = await this.handDetector.estimateHandBounds(image12, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) { if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) {
this.detectedHands = 0; this.detectedHands = 0;
@ -16889,8 +16884,6 @@ var HandPipeline = class {
useFreshBox = true; useFreshBox = true;
} }
const hands = []; const hands = [];
if (config3.hand.skipInitial && this.detectedHands === 0)
this.skipped = 0;
for (let i = 0; i < this.storedBoxes.length; i++) { for (let i = 0; i < this.storedBoxes.length; i++) {
const currentBox = this.storedBoxes[i]; const currentBox = this.storedBoxes[i];
if (!currentBox) if (!currentBox)
@ -16898,8 +16891,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image11.shape[2], palmCenter[1] / image11.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image11, angle, 0, palmCenterNormalized) : image11.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17109,13 +17102,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image11, config3) { async function predict6(image12, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image11.shape[2], height: image11.shape[1]}; const imgSize = {width: image12.shape[2], height: image12.shape[1]};
const resize = tf13.image.resizeBilinear(image11, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17318,20 +17311,17 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image11, config3) { async function predict7(image12, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.videoOptimized && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
skipped3++; skipped3++;
return last3; return last3;
} }
if (config3.videoOptimized)
skipped3 = 0; skipped3 = 0;
else
skipped3 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image11.shape[2], image11.shape[1]]; const outputSize = [image12.shape[2], image12.shape[1]];
const resize = tf14.image.resizeBilinear(image11, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17455,10 +17445,6 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var image_exports = {};
__export(image_exports, {
process: () => process3
});
var tf15 = __toModule(require_tfjs_esm()); var tf15 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
@ -17611,8 +17597,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image11) { this.apply = function(image12) {
_resize(image11.width, image11.height); _resize(image12.width, image12.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17621,7 +17607,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image11); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -19526,16 +19512,17 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`; 2Q==`;
// package.json // package.json
var version = "1.8.5"; var version = "1.9.0";
// src/human.ts // src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _sanity, _checkBackend, _warmupBitmap, _warmupCanvas, _warmupNode; var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
var Human = class { var Human = class {
constructor(userConfig = {}) { constructor(userConfig = {}) {
__privateAdd(this, _numTensors, void 0); __privateAdd(this, _numTensors, void 0);
__privateAdd(this, _analyzeMemoryLeaks, void 0); __privateAdd(this, _analyzeMemoryLeaks, void 0);
__privateAdd(this, _checkSanity, void 0); __privateAdd(this, _checkSanity, void 0);
__privateAdd(this, _firstRun, void 0); __privateAdd(this, _firstRun, void 0);
__privateAdd(this, _lastInputSum, void 0);
this.analyze = (...msg) => { this.analyze = (...msg) => {
if (!__privateGet(this, _analyzeMemoryLeaks)) if (!__privateGet(this, _analyzeMemoryLeaks))
return; return;
@ -19612,6 +19599,19 @@ var Human = class {
this.perf.backend = Math.trunc(now() - timeStamp); this.perf.backend = Math.trunc(now() - timeStamp);
} }
}); });
__privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0)
return true;
const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced);
reduced.dispose();
const sum = sumT.dataSync()[0];
sumT.dispose();
const diff = Math.max(sum, __privateGet(this, _lastInputSum)) / Math.min(sum, __privateGet(this, _lastInputSum)) - 1;
__privateSet(this, _lastInputSum, sum);
return diff < this.config.cacheSensitivity;
});
__privateAdd(this, _warmupBitmap, async () => { __privateAdd(this, _warmupBitmap, async () => {
const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob()); const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob());
let blob; let blob;
@ -19722,6 +19722,7 @@ var Human = class {
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
this.sysinfo = info(); this.sysinfo = info();
__privateSet(this, _lastInputSum, 1);
} }
similarity(embedding1, embedding2) { similarity(embedding1, embedding2) {
return similarity(embedding1, embedding2); return similarity(embedding1, embedding2);
@ -19811,12 +19812,6 @@ var Human = class {
const timeStart = now(); const timeStart = now();
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
let previousVideoOptimized;
if (input && this.config.videoOptimized && typeof window !== "undefined" && typeof WorkerGlobalScope !== "undefined" && (typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement || typeof Image !== "undefined" && input instanceof Image || typeof ImageData !== "undefined" && input instanceof ImageData || typeof ImageBitmap !== "undefined" && image_exports instanceof ImageBitmap)) {
log("disabling video optimization");
previousVideoOptimized = this.config.videoOptimized;
this.config.videoOptimized = false;
}
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process4 = process3(input, this.config);
if (!process4 || !process4.tensor) { if (!process4 || !process4.tensor) {
@ -19826,6 +19821,17 @@ var Human = class {
} }
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
if (!this.perf.frames)
this.perf.frames = 0;
if (!this.perf.cached)
this.perf.cached = 0;
this.perf.frames++;
if (this.config.skipFrame)
this.perf.cached++;
this.perf.changed = Math.trunc(now() - timeStamp);
this.analyze("Check Changed:");
let bodyRes; let bodyRes;
let handRes; let handRes;
let faceRes; let faceRes;
@ -19904,8 +19910,6 @@ var Human = class {
else if (this.perf.gesture) else if (this.perf.gesture)
delete this.perf.gesture; delete this.perf.gesture;
} }
if (previousVideoOptimized)
this.config.videoOptimized = previousVideoOptimized;
this.perf.total = Math.trunc(now() - timeStart); this.perf.total = Math.trunc(now() - timeStart);
this.state = "idle"; this.state = "idle";
const result = { const result = {
@ -19926,8 +19930,6 @@ var Human = class {
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
if (!this.config.warmup || this.config.warmup === "none") if (!this.config.warmup || this.config.warmup === "none")
return {error: "null"}; return {error: "null"};
const save = this.config.videoOptimized;
this.config.videoOptimized = false;
let res; let res;
if (typeof createImageBitmap === "function") if (typeof createImageBitmap === "function")
res = await __privateGet(this, _warmupBitmap).call(this); res = await __privateGet(this, _warmupBitmap).call(this);
@ -19935,7 +19937,6 @@ var Human = class {
res = await __privateGet(this, _warmupCanvas).call(this); res = await __privateGet(this, _warmupCanvas).call(this);
else else
res = await __privateGet(this, _warmupNode).call(this); res = await __privateGet(this, _warmupNode).call(this);
this.config.videoOptimized = save;
const t1 = now(); const t1 = now();
if (this.config.debug) if (this.config.debug)
log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res); log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res);
@ -19946,8 +19947,10 @@ _numTensors = new WeakMap();
_analyzeMemoryLeaks = new WeakMap(); _analyzeMemoryLeaks = new WeakMap();
_checkSanity = new WeakMap(); _checkSanity = new WeakMap();
_firstRun = new WeakMap(); _firstRun = new WeakMap();
_lastInputSum = new WeakMap();
_sanity = new WeakMap(); _sanity = new WeakMap();
_checkBackend = new WeakMap(); _checkBackend = new WeakMap();
_skipFrame = new WeakMap();
_warmupBitmap = new WeakMap(); _warmupBitmap = new WeakMap();
_warmupCanvas = new WeakMap(); _warmupCanvas = new WeakMap();
_warmupNode = new WeakMap(); _warmupNode = new WeakMap();

219
dist/human.node.js vendored
View File

@ -126,8 +126,8 @@ var config = {
wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//", wasmPath: "../node_modules/@tensorflow/tfjs-backend-wasm/dist//",
debug: true, debug: true,
async: true, async: true,
videoOptimized: true,
warmup: "full", warmup: "full",
cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -158,7 +158,6 @@ var config = {
rotation: false, rotation: false,
maxDetected: 10, maxDetected: 10,
skipFrames: 21, skipFrames: 21,
skipInitial: false,
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.1, iouThreshold: 0.1,
return: false return: false
@ -194,7 +193,6 @@ var config = {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 12,
skipInitial: false,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -340,16 +338,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image11, cropSize) { function cutBoxFromImageAndResize(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image11, boxes, [0], cropSize); return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -3896,10 +3894,10 @@ var Pipeline = class {
const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2]; const rightEyeZ = rawCoords[eyeLandmarks.rightBounds[0]][2];
return leftEyeZ - rightEyeZ; return leftEyeZ - rightEyeZ;
} }
getEyeBox(rawCoords, face4, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) { getEyeBox(rawCoords, face5, eyeInnerCornerIndex, eyeOuterCornerIndex, flip = false) {
const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge)); const box4 = squarifyBox(enlargeBox(calculateLandmarksBoundingBox([rawCoords[eyeInnerCornerIndex], rawCoords[eyeOuterCornerIndex]]), this.irisEnlarge));
const boxSize = getBoxSize(box4); const boxSize = getBoxSize(box4);
let crop = tf4.image.cropAndResize(face4, [[ let crop = tf4.image.cropAndResize(face5, [[
box4.startPoint[1] / this.meshSize, box4.startPoint[1] / this.meshSize,
box4.startPoint[0] / this.meshSize, box4.startPoint[0] / this.meshSize,
box4.endPoint[1] / this.meshSize, box4.endPoint[1] / this.meshSize,
@ -3941,13 +3939,13 @@ var Pipeline = class {
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.face.detector.skipFrames || !config3.face.mesh.enabled || !config3.skipFrame) {
detector = await this.boundingBoxDetector.getBoundingBoxes(input); detector = await this.boundingBoxDetector.getBoundingBoxes(input);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (!config3.videoOptimized || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) { if (!config3.skipFrame || detector && detector.boxes && (!config3.face.mesh.enabled || detector.boxes.length !== this.detectedFaces && this.detectedFaces !== config3.face.detector.maxDetected)) {
this.storedBoxes = []; this.storedBoxes = [];
this.detectedFaces = 0; this.detectedFaces = 0;
for (const possible of detector.boxes) { for (const possible of detector.boxes) {
@ -3956,8 +3954,6 @@ var Pipeline = class {
if (this.storedBoxes.length > 0) if (this.storedBoxes.length > 0)
useFreshBox = true; useFreshBox = true;
} }
if (config3.face.detector.skipInitial && this.detectedFaces === 0)
this.skipped = 0;
if (useFreshBox) { if (useFreshBox) {
if (!detector || !detector.boxes || detector.boxes.length === 0) { if (!detector || !detector.boxes || detector.boxes.length === 0) {
this.storedBoxes = []; this.storedBoxes = [];
@ -3981,7 +3977,7 @@ var Pipeline = class {
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => { const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
let face4; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
@ -3992,16 +3988,16 @@ var Pipeline = class {
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.boxSize, this.boxSize]).div(255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.meshSize, this.meshSize]).div(255);
else else
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, clonedImage, [this.boxSize, this.boxSize]).div(255);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { const prediction2 = {
@ -4010,11 +4006,11 @@ var Pipeline = class {
faceConfidence: null, faceConfidence: null,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
confidence: box4.confidence, confidence: box4.confidence,
image: face4 image: face5
}; };
return prediction2; return prediction2;
} }
const [, confidence, contourCoords] = this.meshDetector.execute(face4); const [, confidence, contourCoords] = this.meshDetector.execute(face5);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = confidence.dataSync()[0];
if (faceConfidence < config3.face.detector.minConfidence) { if (faceConfidence < config3.face.detector.minConfidence) {
this.storedBoxes[i].confidence = faceConfidence; this.storedBoxes[i].confidence = faceConfidence;
@ -4023,8 +4019,8 @@ var Pipeline = class {
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
let rawCoords = coordsReshaped.arraySync(); let rawCoords = coordsReshaped.arraySync();
if (config3.face.iris.enabled) { if (config3.face.iris.enabled) {
const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); const {box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face4, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]); const {box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop} = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop])); const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync(); const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3); const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
@ -4055,14 +4051,14 @@ var Pipeline = class {
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized); const rotatedImage = tf4.image.rotateWithOffset(input.toFloat(), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter); rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255); face5 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
} }
const prediction = { const prediction = {
mesh, mesh,
box: box4, box: box4,
faceConfidence, faceConfidence,
boxConfidence: box4.confidence, boxConfidence: box4.confidence,
image: face4 image: face5
}; };
const storedBox = squarifyBox(box4); const storedBox = squarifyBox(box4);
storedBox.confidence = box4.confidence; storedBox.confidence = box4.confidence;
@ -4164,6 +4160,7 @@ var tf6 = __toModule(require_tfjs_esm());
var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]; var annotations = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"];
var model; var model;
var last = []; var last = [];
var lastCount = 0;
var skipped = Number.MAX_SAFE_INTEGER; var skipped = Number.MAX_SAFE_INTEGER;
var rgb = [0.2989, 0.587, 0.114]; var rgb = [0.2989, 0.587, 0.114];
async function load3(config3) { async function load3(config3) {
@ -4177,19 +4174,16 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image11, config3) { async function predict2(image12, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.videoOptimized && last.length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
skipped++; skipped++;
return last; return last[idx];
} }
if (config3.videoOptimized)
skipped = 0; skipped = 0;
else
skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image11, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4216,7 +4210,8 @@ async function predict2(image11, config3) {
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); normalize.dispose();
last = obj; last[idx] = obj;
lastCount = count2;
resolve(obj); resolve(obj);
}); });
} }
@ -4232,7 +4227,8 @@ __export(faceres_exports, {
}); });
var tf7 = __toModule(require_tfjs_esm()); var tf7 = __toModule(require_tfjs_esm());
var model2; var model2;
var last2 = {age: 0}; var last2 = [];
var lastCount2 = 0;
var skipped2 = Number.MAX_SAFE_INTEGER; var skipped2 = Number.MAX_SAFE_INTEGER;
async function load4(config3) { async function load4(config3) {
if (!model2) { if (!model2) {
@ -4270,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image11 = tf7.tidy(() => { const image12 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4279,21 +4275,19 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image11; return image12;
} }
async function predict3(image11, config3) { async function predict3(image12, config3, idx, count2) {
var _a, _b;
if (!model2) if (!model2)
return null; return null;
if (skipped2 < config3.face.description.skipFrames && config3.videoOptimized && last2.age && last2.age > 0) { if (skipped2 < config3.face.description.skipFrames && config3.skipFrame && lastCount2 === count2 && ((_a = last2[idx]) == null ? void 0 : _a.age) && ((_b = last2[idx]) == null ? void 0 : _b.age) > 0) {
skipped2++; skipped2++;
return last2; return last2;
} }
if (config3.videoOptimized)
skipped2 = 0; skipped2 = 0;
else
skipped2 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image11); const enhanced = enhance(image12);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4320,13 +4314,14 @@ async function predict3(image11, config3) {
}); });
resT.forEach((t) => tf7.dispose(t)); resT.forEach((t) => tf7.dispose(t));
} }
last2 = obj; last2[idx] = obj;
lastCount2 = count2;
resolve(obj); resolve(obj);
}); });
} }
// src/faceall.ts // src/face.ts
var calculateFaceAngle = (face4, image_size) => { var calculateFaceAngle = (face5, image_size) => {
const degrees = (theta) => theta * 180 / Math.PI; const degrees = (theta) => theta * 180 / Math.PI;
const normalize = (v) => { const normalize = (v) => {
const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
@ -4378,10 +4373,10 @@ var calculateFaceAngle = (face4, image_size) => {
}; };
return angle2; return angle2;
}; };
const mesh = face4.meshRaw; const mesh = face5.meshRaw;
if (!mesh || mesh.length < 300) if (!mesh || mesh.length < 300)
return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]}; return {angle: {pitch: 0, yaw: 0, roll: 0}, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1]};
const size = Math.max(face4.boxRaw[2] * image_size[0], face4.boxRaw[3] * image_size[1]) / 1.5; const size = Math.max(face5.boxRaw[2] * image_size[0], face5.boxRaw[3] * image_size[1]) / 1.5;
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
pt[0] * image_size[0] / size, pt[0] * image_size[0] / size,
pt[1] * image_size[1] / size, pt[1] * image_size[1] / size,
@ -4406,7 +4401,7 @@ var calculateFaceAngle = (face4, image_size) => {
return {angle, matrix}; return {angle, matrix};
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {
var _a, _b, _c, _d, _e, _f; var _a, _b, _c, _d, _e, _f, _g, _h;
let timeStamp; let timeStamp;
let ageRes; let ageRes;
let genderRes; let genderRes;
@ -4420,30 +4415,30 @@ var detectFace = async (parent, input) => {
parent.perf.face = Math.trunc(now() - timeStamp); parent.perf.face = Math.trunc(now() - timeStamp);
if (!faces) if (!faces)
return []; return [];
for (const face4 of faces) { for (let i = 0; i < faces.length; i++) {
parent.analyze("Get Face"); parent.analyze("Get Face");
if (!face4.image || face4.image.isDisposedInternal) { if (!faces[i].image || faces[i].image.isDisposedInternal) {
log("Face object is disposed:", face4.image); log("Face object is disposed:", faces[i].image);
continue; continue;
} }
const rotation = calculateFaceAngle(face4, [input.shape[2], input.shape[1]]); const rotation = calculateFaceAngle(faces[i], [input.shape[2], input.shape[1]]);
parent.analyze("Start Emotion:"); parent.analyze("Start Emotion:");
if (parent.config.async) { if (parent.config.async) {
emotionRes = parent.config.face.emotion.enabled ? predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? predict2(faces[i].image, parent.config, i, faces.length) : {};
} else { } else {
parent.state = "run:emotion"; parent.state = "run:emotion";
timeStamp = now(); timeStamp = now();
emotionRes = parent.config.face.emotion.enabled ? await predict2(face4.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? await predict2(faces[i].image, parent.config, i, faces.length) : {};
parent.perf.emotion = Math.trunc(now() - timeStamp); parent.perf.emotion = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Emotion:"); parent.analyze("End Emotion:");
parent.analyze("Start Description:"); parent.analyze("Start Description:");
if (parent.config.async) { if (parent.config.async) {
descRes = parent.config.face.description.enabled ? predict3(face4, parent.config) : []; descRes = parent.config.face.description.enabled ? predict3(faces[i], parent.config, i, faces.length) : [];
} else { } else {
parent.state = "run:description"; parent.state = "run:description";
timeStamp = now(); timeStamp = now();
descRes = parent.config.face.description.enabled ? await predict3(face4.image, parent.config) : []; descRes = parent.config.face.description.enabled ? await predict3(faces[i].image, parent.config, i, faces.length) : [];
parent.perf.embedding = Math.trunc(now() - timeStamp); parent.perf.embedding = Math.trunc(now() - timeStamp);
} }
parent.analyze("End Description:"); parent.analyze("End Description:");
@ -4451,13 +4446,13 @@ var detectFace = async (parent, input) => {
[ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]); [ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]);
} }
parent.analyze("Finish Face:"); parent.analyze("Finish Face:");
if (!parent.config.face.iris.enabled && ((_a = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _a.leftEyeIris) && ((_b = face4 == null ? void 0 : face4.annotations) == null ? void 0 : _b.rightEyeIris)) { if (!parent.config.face.iris.enabled && ((_b = (_a = faces[i]) == null ? void 0 : _a.annotations) == null ? void 0 : _b.leftEyeIris) && ((_d = (_c = faces[i]) == null ? void 0 : _c.annotations) == null ? void 0 : _d.rightEyeIris)) {
delete face4.annotations.leftEyeIris; delete faces[i].annotations.leftEyeIris;
delete face4.annotations.rightEyeIris; delete faces[i].annotations.rightEyeIris;
} }
const irisSize = ((_c = face4.annotations) == null ? void 0 : _c.leftEyeIris) && ((_d = face4.annotations) == null ? void 0 : _d.rightEyeIris) ? 11.7 * Math.max(Math.abs(face4.annotations.leftEyeIris[3][0] - face4.annotations.leftEyeIris[1][0]), Math.abs(face4.annotations.rightEyeIris[4][1] - face4.annotations.rightEyeIris[2][1])) : 0; const irisSize = ((_e = faces[i].annotations) == null ? void 0 : _e.leftEyeIris) && ((_f = faces[i].annotations) == null ? void 0 : _f.rightEyeIris) ? 11.7 * Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) : 0;
faceRes.push({ faceRes.push({
...face4, ...faces[i],
age: descRes.age, age: descRes.age,
gender: descRes.gender, gender: descRes.gender,
genderConfidence: descRes.genderConfidence, genderConfidence: descRes.genderConfidence,
@ -4465,9 +4460,9 @@ var detectFace = async (parent, input) => {
emotion: emotionRes, emotion: emotionRes,
iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0, iris: irisSize !== 0 ? Math.trunc(irisSize) / 100 : 0,
rotation, rotation,
tensor: parent.config.face.detector.return ? (_e = face4.image) == null ? void 0 : _e.squeeze() : null tensor: parent.config.face.detector.return ? (_g = faces[i].image) == null ? void 0 : _g.squeeze() : null
}); });
(_f = face4.image) == null ? void 0 : _f.dispose(); (_h = faces[i].image) == null ? void 0 : _h.dispose();
parent.analyze("End Face"); parent.analyze("End Face");
} }
parent.analyze("End FaceMesh:"); parent.analyze("End FaceMesh:");
@ -4849,16 +4844,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image11, cropSize) { function cutBoxFromImageAndResize2(box4, image12, cropSize) {
const h = image11.shape[1]; const h = image12.shape[1];
const w = image11.shape[2]; const w = image12.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image11, boxes, [0], cropSize); return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16725,9 +16720,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image11 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image11, config3); const predictions = await this.getBoxes(image12, config3);
image11.dispose(); image12.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16872,14 +16867,14 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image11, config3) { async estimateHands(image12, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.videoOptimized) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image11, config3); boxes = await this.handDetector.estimateHandBounds(image12, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.videoOptimized) if (config3.skipFrame)
this.skipped++; this.skipped++;
if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) { if (boxes && boxes.length > 0 && (boxes.length !== this.detectedHands && this.detectedHands !== config3.hand.maxDetected || !config3.hand.landmarks)) {
this.detectedHands = 0; this.detectedHands = 0;
@ -16888,8 +16883,6 @@ var HandPipeline = class {
useFreshBox = true; useFreshBox = true;
} }
const hands = []; const hands = [];
if (config3.hand.skipInitial && this.detectedHands === 0)
this.skipped = 0;
for (let i = 0; i < this.storedBoxes.length; i++) { for (let i = 0; i < this.storedBoxes.length; i++) {
const currentBox = this.storedBoxes[i]; const currentBox = this.storedBoxes[i];
if (!currentBox) if (!currentBox)
@ -16897,8 +16890,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image11.shape[2], palmCenter[1] / image11.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image11, angle, 0, palmCenterNormalized) : image11.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17108,13 +17101,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image11, config3) { async function predict6(image12, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image11.shape[2], height: image11.shape[1]}; const imgSize = {width: image12.shape[2], height: image12.shape[1]};
const resize = tf13.image.resizeBilinear(image11, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17317,20 +17310,17 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image11, config3) { async function predict7(image12, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.videoOptimized && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
skipped3++; skipped3++;
return last3; return last3;
} }
if (config3.videoOptimized)
skipped3 = 0; skipped3 = 0;
else
skipped3 = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image11.shape[2], image11.shape[1]]; const outputSize = [image12.shape[2], image12.shape[1]];
const resize = tf14.image.resizeBilinear(image11, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17454,10 +17444,6 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var image_exports = {};
__export(image_exports, {
process: () => process3
});
var tf15 = __toModule(require_tfjs_esm()); var tf15 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
@ -17610,8 +17596,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image11) { this.apply = function(image12) {
_resize(image11.width, image11.height); _resize(image12.width, image12.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17620,7 +17606,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image11); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -19525,16 +19511,17 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`; 2Q==`;
// package.json // package.json
var version = "1.8.5"; var version = "1.9.0";
// src/human.ts // src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _sanity, _checkBackend, _warmupBitmap, _warmupCanvas, _warmupNode; var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
var Human = class { var Human = class {
constructor(userConfig = {}) { constructor(userConfig = {}) {
__privateAdd(this, _numTensors, void 0); __privateAdd(this, _numTensors, void 0);
__privateAdd(this, _analyzeMemoryLeaks, void 0); __privateAdd(this, _analyzeMemoryLeaks, void 0);
__privateAdd(this, _checkSanity, void 0); __privateAdd(this, _checkSanity, void 0);
__privateAdd(this, _firstRun, void 0); __privateAdd(this, _firstRun, void 0);
__privateAdd(this, _lastInputSum, void 0);
this.analyze = (...msg) => { this.analyze = (...msg) => {
if (!__privateGet(this, _analyzeMemoryLeaks)) if (!__privateGet(this, _analyzeMemoryLeaks))
return; return;
@ -19611,6 +19598,19 @@ var Human = class {
this.perf.backend = Math.trunc(now() - timeStamp); this.perf.backend = Math.trunc(now() - timeStamp);
} }
}); });
__privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0)
return true;
const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced);
reduced.dispose();
const sum = sumT.dataSync()[0];
sumT.dispose();
const diff = Math.max(sum, __privateGet(this, _lastInputSum)) / Math.min(sum, __privateGet(this, _lastInputSum)) - 1;
__privateSet(this, _lastInputSum, sum);
return diff < this.config.cacheSensitivity;
});
__privateAdd(this, _warmupBitmap, async () => { __privateAdd(this, _warmupBitmap, async () => {
const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob()); const b64toBlob = (base64, type = "application/octet-stream") => fetch(`data:${type};base64,${base64}`).then((res2) => res2.blob());
let blob; let blob;
@ -19721,6 +19721,7 @@ var Human = class {
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
this.sysinfo = info(); this.sysinfo = info();
__privateSet(this, _lastInputSum, 1);
} }
similarity(embedding1, embedding2) { similarity(embedding1, embedding2) {
return similarity(embedding1, embedding2); return similarity(embedding1, embedding2);
@ -19810,12 +19811,6 @@ var Human = class {
const timeStart = now(); const timeStart = now();
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
let previousVideoOptimized;
if (input && this.config.videoOptimized && typeof window !== "undefined" && typeof WorkerGlobalScope !== "undefined" && (typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement || typeof Image !== "undefined" && input instanceof Image || typeof ImageData !== "undefined" && input instanceof ImageData || typeof ImageBitmap !== "undefined" && image_exports instanceof ImageBitmap)) {
log("disabling video optimization");
previousVideoOptimized = this.config.videoOptimized;
this.config.videoOptimized = false;
}
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process4 = process3(input, this.config);
if (!process4 || !process4.tensor) { if (!process4 || !process4.tensor) {
@ -19825,6 +19820,17 @@ var Human = class {
} }
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
if (!this.perf.frames)
this.perf.frames = 0;
if (!this.perf.cached)
this.perf.cached = 0;
this.perf.frames++;
if (this.config.skipFrame)
this.perf.cached++;
this.perf.changed = Math.trunc(now() - timeStamp);
this.analyze("Check Changed:");
let bodyRes; let bodyRes;
let handRes; let handRes;
let faceRes; let faceRes;
@ -19903,8 +19909,6 @@ var Human = class {
else if (this.perf.gesture) else if (this.perf.gesture)
delete this.perf.gesture; delete this.perf.gesture;
} }
if (previousVideoOptimized)
this.config.videoOptimized = previousVideoOptimized;
this.perf.total = Math.trunc(now() - timeStart); this.perf.total = Math.trunc(now() - timeStart);
this.state = "idle"; this.state = "idle";
const result = { const result = {
@ -19925,8 +19929,6 @@ var Human = class {
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
if (!this.config.warmup || this.config.warmup === "none") if (!this.config.warmup || this.config.warmup === "none")
return {error: "null"}; return {error: "null"};
const save = this.config.videoOptimized;
this.config.videoOptimized = false;
let res; let res;
if (typeof createImageBitmap === "function") if (typeof createImageBitmap === "function")
res = await __privateGet(this, _warmupBitmap).call(this); res = await __privateGet(this, _warmupBitmap).call(this);
@ -19934,7 +19936,6 @@ var Human = class {
res = await __privateGet(this, _warmupCanvas).call(this); res = await __privateGet(this, _warmupCanvas).call(this);
else else
res = await __privateGet(this, _warmupNode).call(this); res = await __privateGet(this, _warmupNode).call(this);
this.config.videoOptimized = save;
const t1 = now(); const t1 = now();
if (this.config.debug) if (this.config.debug)
log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res); log("Warmup", this.config.warmup, Math.round(t1 - t0), "ms", res);
@ -19945,8 +19946,10 @@ _numTensors = new WeakMap();
_analyzeMemoryLeaks = new WeakMap(); _analyzeMemoryLeaks = new WeakMap();
_checkSanity = new WeakMap(); _checkSanity = new WeakMap();
_firstRun = new WeakMap(); _firstRun = new WeakMap();
_lastInputSum = new WeakMap();
_sanity = new WeakMap(); _sanity = new WeakMap();
_checkBackend = new WeakMap(); _checkBackend = new WeakMap();
_skipFrame = new WeakMap();
_warmupBitmap = new WeakMap(); _warmupBitmap = new WeakMap();
_warmupCanvas = new WeakMap(); _warmupCanvas = new WeakMap();
_warmupNode = new WeakMap(); _warmupNode = new WeakMap();

59239
dist/tfjs.esm.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/tfjs.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "1.8.5", "version": "1.9.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",

5
src/age/age.ts
View File

@ -16,12 +16,11 @@ export async function load(config) {
export async function predict(image, config) { export async function predict(image, config) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.age.skipFrames) && config.videoOptimized && last.age && (last.age > 0)) { if ((skipped < config.face.age.skipFrames) && config.skipFrame && last.age && (last.age > 0)) {
skipped++; skipped++;
return last; return last;
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const enhance = tf.mul(resize, [255.0]); const enhance = tf.mul(resize, [255.0]);

8
src/blazeface/facepipeline.ts
View File

@ -155,14 +155,14 @@ export class Pipeline {
let useFreshBox = false; let useFreshBox = false;
// run new detector every skipFrames unless we only want box to start with // run new detector every skipFrames unless we only want box to start with
let detector; let detector;
if ((this.skipped === 0) || (this.skipped > config.face.detector.skipFrames) || !config.face.mesh.enabled || !config.videoOptimized) { if ((this.skipped === 0) || (this.skipped > config.face.detector.skipFrames) || !config.face.mesh.enabled || !config.skipFrame) {
detector = await this.boundingBoxDetector.getBoundingBoxes(input); detector = await this.boundingBoxDetector.getBoundingBoxes(input);
this.skipped = 0; this.skipped = 0;
} }
if (config.videoOptimized) this.skipped++; if (config.skipFrame) this.skipped++;
// if detector result count doesn't match current working set, use it to reset current working set // if detector result count doesn't match current working set, use it to reset current working set
if (!config.videoOptimized || (detector && detector.boxes && (!config.face.mesh.enabled || (detector.boxes.length !== this.detectedFaces) && (this.detectedFaces !== config.face.detector.maxDetected)))) { if (!config.skipFrame || (detector && detector.boxes && (!config.face.mesh.enabled || (detector.boxes.length !== this.detectedFaces) && (this.detectedFaces !== config.face.detector.maxDetected)))) {
this.storedBoxes = []; this.storedBoxes = [];
this.detectedFaces = 0; this.detectedFaces = 0;
for (const possible of detector.boxes) { for (const possible of detector.boxes) {
@ -171,8 +171,6 @@ export class Pipeline {
if (this.storedBoxes.length > 0) useFreshBox = true; if (this.storedBoxes.length > 0) useFreshBox = true;
} }
if (config.face.detector.skipInitial && this.detectedFaces === 0) this.skipped = 0;
if (useFreshBox) { if (useFreshBox) {
if (!detector || !detector.boxes || (detector.boxes.length === 0)) { if (!detector || !detector.boxes || (detector.boxes.length === 0)) {
this.storedBoxes = []; this.storedBoxes = [];

49
src/config.ts
View File

@ -19,13 +19,6 @@ export interface Config {
/** Perform model loading and inference concurrently or sequentially */ /** Perform model loading and inference concurrently or sequentially */
async: boolean, async: boolean,
/** Perform additional optimizations when input is video,
* - must be disabled for images
* - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
* - skips boundary detection for every `skipFrames` frames specified for each model
* - while maintaining in-box detection since objects don't change definition as fast */
videoOptimized: boolean,
/** What to use for `human.warmup()` /** What to use for `human.warmup()`
* - warmup pre-initializes all models for faster inference but can take significant time on startup * - warmup pre-initializes all models for faster inference but can take significant time on startup
* - only used for `webgl` and `humangl` backends * - only used for `webgl` and `humangl` backends
@ -37,6 +30,12 @@ export interface Config {
*/ */
modelBasePath: string, modelBasePath: string,
/** Cache sensitivity
* - values 0..1 where 0.01 means reset cache if input changed more than 1%
* - set to 0 to disable caching
*/
cacheSensitivity: number;
/** Run input through image filters before inference /** Run input through image filters before inference
* - image filters run with near-zero latency as they are executed on the GPU * - image filters run with near-zero latency as they are executed on the GPU
*/ */
@ -101,8 +100,6 @@ export interface Config {
* - iouThreshold: ammount of overlap between two detected objects before one object is removed * - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
* - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
* - return: return extracted face as tensor for futher user processing * - return: return extracted face as tensor for futher user processing
*/ */
face: { face: {
@ -112,7 +109,6 @@ export interface Config {
rotation: boolean, rotation: boolean,
maxDetected: number, maxDetected: number,
skipFrames: number, skipFrames: number,
skipInitial: boolean,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
return: boolean, return: boolean,
@ -160,14 +156,11 @@ export interface Config {
* - iouThreshold: ammount of overlap between two detected objects before one object is removed * - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
* - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
* - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
*/ */
hand: { hand: {
enabled: boolean, enabled: boolean,
rotation: boolean, rotation: boolean,
skipFrames: number, skipFrames: number,
skipInitial: boolean,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
maxDetected: number, maxDetected: number,
@ -186,7 +179,6 @@ export interface Config {
* - minConfidence: minimum score that detection must have to return as valid object * - minConfidence: minimum score that detection must have to return as valid object
* - iouThreshold: ammount of overlap between two detected objects before one object is removed * - iouThreshold: ammount of overlap between two detected objects before one object is removed
* - maxDetected: maximum number of detections to return * - maxDetected: maximum number of detections to return
* - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
*/ */
object: { object: {
enabled: boolean, enabled: boolean,
@ -205,14 +197,13 @@ const config: Config = {
wasmPath: '../node_modules/@tensorflow/tfjs-backend-wasm/dist//', // path for wasm binaries, only used for backend: wasm wasmPath: '../node_modules/@tensorflow/tfjs-backend-wasm/dist//', // path for wasm binaries, only used for backend: wasm
debug: true, // print additional status messages to console debug: true, // print additional status messages to console
async: true, // execute enabled models in parallel async: true, // execute enabled models in parallel
videoOptimized: true, // perform additional optimizations when input is video,
// automatically disabled for Image, ImageData, ImageBitmap
// skips boundary detection for every n frames
// while maintaining in-box detection since objects cannot move that fast
warmup: 'full', // what to use for human.warmup(), can be 'none', 'face', 'full' warmup: 'full', // what to use for human.warmup(), can be 'none', 'face', 'full'
// warmup pre-initializes all models for faster inference but can take // warmup pre-initializes all models for faster inference but can take
// significant time on startup // significant time on startup
// only used for `webgl` and `humangl` backends // only used for `webgl` and `humangl` backends
cacheSensitivity: 0.005, // cache sensitivity
// values 0..1 where 0.01 means reset cache if input changed more than 1%
// set to 0 to disable caching
filter: { // run input through image filters before inference filter: { // run input through image filters before inference
// image filters run with near-zero latency as they are executed on the GPU // image filters run with near-zero latency as they are executed on the GPU
enabled: true, // enable image pre-processing filters enabled: true, // enable image pre-processing filters
@ -254,13 +245,11 @@ const config: Config = {
// this parameter is not valid in nodejs // this parameter is not valid in nodejs
maxDetected: 10, // maximum number of faces detected in the input maxDetected: 10, // maximum number of faces detected in the input
// should be set to the minimum number for performance // should be set to the minimum number for performance
skipFrames: 21, // how many frames to go without re-running the face bounding box detector skipFrames: 21, // how many max frames to go without re-running the face bounding box detector
// only used for video inputs // only used when cacheSensitivity is not zero
// e.g., if model is running st 25 FPS, we can re-use existing bounding // e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated face analysis as the head probably hasn't moved much // box for updated face analysis as the head probably hasn't moved much
// in short time (10 * 1/25 = 0.25 sec) // in short time (10 * 1/25 = 0.25 sec)
skipInitial: false, // if previous detection resulted in no faces detected,
// should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
return: false, // return extracted face as tensor return: false, // return extracted face as tensor
@ -282,15 +271,16 @@ const config: Config = {
// recommended to enable detector.rotation and mesh.enabled // recommended to enable detector.rotation and mesh.enabled
modelPath: 'faceres.json', // face description model modelPath: 'faceres.json', // face description model
// can be either absolute path or relative to modelBasePath // can be either absolute path or relative to modelBasePath
skipFrames: 31, // how many frames to go without re-running the detector skipFrames: 31, // how many max frames to go without re-running the detector
// only used for video inputs // only used when cacheSensitivity is not zero
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
}, },
emotion: { emotion: {
enabled: true, enabled: true,
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 32, // how many frames to go without re-running the detector skipFrames: 32, // how max many frames to go without re-running the detector
// only used when cacheSensitivity is not zero
modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath
}, },
}, },
@ -309,13 +299,11 @@ const config: Config = {
enabled: true, enabled: true,
rotation: false, // use best-guess rotated hand image or just box with rotation as-is rotation: false, // use best-guess rotated hand image or just box with rotation as-is
// false means higher performance, but incorrect finger mapping if hand is inverted // false means higher performance, but incorrect finger mapping if hand is inverted
skipFrames: 12, // how many frames to go without re-running the hand bounding box detector skipFrames: 12, // how many max frames to go without re-running the hand bounding box detector
// only used for video inputs // only used when cacheSensitivity is not zero
// e.g., if model is running st 25 FPS, we can re-use existing bounding // e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated hand skeleton analysis as the hand probably // box for updated hand skeleton analysis as the hand probably
// hasn't moved much in short time (10 * 1/25 = 0.25 sec) // hasn't moved much in short time (10 * 1/25 = 0.25 sec)
skipInitial: false, // if previous detection resulted in no hands detected,
// should skipFrames be reset immediately to force new detection cycle
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
maxDetected: 2, // maximum number of hands detected in the input maxDetected: 2, // maximum number of hands detected in the input
@ -335,7 +323,8 @@ const config: Config = {
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
maxDetected: 10, // maximum number of objects detected in the input maxDetected: 10, // maximum number of objects detected in the input
skipFrames: 41, // how many frames to go without re-running the detector skipFrames: 41, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero
}, },
}; };
export { config as defaults }; export { config as defaults };

5
src/efficientpose/efficientpose.ts
View File

@ -39,12 +39,11 @@ function max2d(inputs, minScore) {
export async function predict(image, config) { export async function predict(image, config) {
if (!model) return null; if (!model) return null;
if ((skipped < config.body.skipFrames) && config.videoOptimized && Object.keys(keypoints).length > 0) { if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
skipped++; skipped++;
return keypoints; return keypoints;
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const tensor = tf.tidy(() => { const tensor = tf.tidy(() => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);

16
src/emotion/emotion.ts
View File

@ -3,7 +3,9 @@ import * as tf from '../../dist/tfjs.esm.js';
const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
let model; let model;
let last: Array<{ score: number, emotion: string }> = []; // let last: Array<{ score: number, emotion: string }> = [];
const last: Array<Array<{ score: number, emotion: string }>> = [];
let lastCount = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
// tuning values // tuning values
@ -18,14 +20,13 @@ export async function load(config) {
return model; return model;
} }
export async function predict(image, config) { export async function predict(image, config, idx, count) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.emotion.skipFrames) && config.videoOptimized && (last.length > 0)) { if ((skipped < config.face.emotion.skipFrames) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
skipped++; skipped++;
return last; return last[idx];
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf.split(resize, 3, 3); const [red, green, blue] = tf.split(resize, 3, 3);
@ -54,7 +55,8 @@ export async function predict(image, config) {
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
} }
normalize.dispose(); normalize.dispose();
last = obj; last[idx] = obj;
lastCount = count;
resolve(obj); resolve(obj);
}); });
} }

36
src/faceall.ts → src/face.ts
View File

@ -57,8 +57,7 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu
const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1); const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1);
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const angle = { const angle = {
// values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center
// value of 0 means center
// pitch is face move up/down // pitch is face move up/down
pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face
// yaw is face turn left/right // yaw is face turn left/right
@ -134,25 +133,26 @@ export const detectFace = async (parent, input): Promise<any> => {
const faces = await facemesh.predict(input, parent.config); const faces = await facemesh.predict(input, parent.config);
parent.perf.face = Math.trunc(now() - timeStamp); parent.perf.face = Math.trunc(now() - timeStamp);
if (!faces) return []; if (!faces) return [];
for (const face of faces) { // for (const face of faces) {
for (let i = 0; i < faces.length; i++) {
parent.analyze('Get Face'); parent.analyze('Get Face');
// is something went wrong, skip the face // is something went wrong, skip the face
if (!face.image || face.image.isDisposedInternal) { if (!faces[i].image || faces[i].image.isDisposedInternal) {
log('Face object is disposed:', face.image); log('Face object is disposed:', faces[i].image);
continue; continue;
} }
const rotation = calculateFaceAngle(face, [input.shape[2], input.shape[1]]); const rotation = calculateFaceAngle(faces[i], [input.shape[2], input.shape[1]]);
// run emotion, inherits face from blazeface // run emotion, inherits face from blazeface
parent.analyze('Start Emotion:'); parent.analyze('Start Emotion:');
if (parent.config.async) { if (parent.config.async) {
emotionRes = parent.config.face.emotion.enabled ? emotion.predict(face.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].image, parent.config, i, faces.length) : {};
} else { } else {
parent.state = 'run:emotion'; parent.state = 'run:emotion';
timeStamp = now(); timeStamp = now();
emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(face.image, parent.config) : {}; emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].image, parent.config, i, faces.length) : {};
parent.perf.emotion = Math.trunc(now() - timeStamp); parent.perf.emotion = Math.trunc(now() - timeStamp);
} }
parent.analyze('End Emotion:'); parent.analyze('End Emotion:');
@ -160,11 +160,11 @@ export const detectFace = async (parent, input): Promise<any> => {
// run emotion, inherits face from blazeface // run emotion, inherits face from blazeface
parent.analyze('Start Description:'); parent.analyze('Start Description:');
if (parent.config.async) { if (parent.config.async) {
descRes = parent.config.face.description.enabled ? faceres.predict(face, parent.config) : []; descRes = parent.config.face.description.enabled ? faceres.predict(faces[i], parent.config, i, faces.length) : [];
} else { } else {
parent.state = 'run:description'; parent.state = 'run:description';
timeStamp = now(); timeStamp = now();
descRes = parent.config.face.description.enabled ? await faceres.predict(face.image, parent.config) : []; descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].image, parent.config, i, faces.length) : [];
parent.perf.embedding = Math.trunc(now() - timeStamp); parent.perf.embedding = Math.trunc(now() - timeStamp);
} }
parent.analyze('End Description:'); parent.analyze('End Description:');
@ -178,18 +178,18 @@ export const detectFace = async (parent, input): Promise<any> => {
// calculate iris distance // calculate iris distance
// iris: array[ center, left, top, right, bottom] // iris: array[ center, left, top, right, bottom]
if (!parent.config.face.iris.enabled && face?.annotations?.leftEyeIris && face?.annotations?.rightEyeIris) { if (!parent.config.face.iris.enabled && faces[i]?.annotations?.leftEyeIris && faces[i]?.annotations?.rightEyeIris) {
delete face.annotations.leftEyeIris; delete faces[i].annotations.leftEyeIris;
delete face.annotations.rightEyeIris; delete faces[i].annotations.rightEyeIris;
} }
const irisSize = (face.annotations?.leftEyeIris && face.annotations?.rightEyeIris) const irisSize = (faces[i].annotations?.leftEyeIris && faces[i].annotations?.rightEyeIris)
/* average human iris size is 11.7mm */ /* average human iris size is 11.7mm */
? 11.7 * Math.max(Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]), Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1])) ? 11.7 * Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1]))
: 0; : 0;
// combine results // combine results
faceRes.push({ faceRes.push({
...face, ...faces[i],
age: descRes.age, age: descRes.age,
gender: descRes.gender, gender: descRes.gender,
genderConfidence: descRes.genderConfidence, genderConfidence: descRes.genderConfidence,
@ -197,10 +197,10 @@ export const detectFace = async (parent, input): Promise<any> => {
emotion: emotionRes, emotion: emotionRes,
iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0, iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0,
rotation, rotation,
tensor: parent.config.face.detector.return ? face.image?.squeeze() : null, tensor: parent.config.face.detector.return ? faces[i].image?.squeeze() : null,
}); });
// dispose original face tensor // dispose original face tensor
face.image?.dispose(); faces[i].image?.dispose();
parent.analyze('End Face'); parent.analyze('End Face');
} }

13
src/faceres/faceres.ts
View File

@ -2,7 +2,8 @@ import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
let model; let model;
let last = { age: 0 }; const last: Array<{ age: number}> = [];
let lastCount = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
type Tensor = typeof tf.Tensor; type Tensor = typeof tf.Tensor;
@ -94,14 +95,13 @@ export function enhance(input): Tensor {
return image; return image;
} }
export async function predict(image, config) { export async function predict(image, config, idx, count) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.description.skipFrames) && config.videoOptimized && last.age && (last.age > 0)) { if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
skipped++; skipped++;
return last; return last;
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image); const enhanced = enhance(image);
@ -136,7 +136,8 @@ export async function predict(image, config) {
resT.forEach((t) => tf.dispose(t)); resT.forEach((t) => tf.dispose(t));
} }
last = obj; last[idx] = obj;
lastCount = count;
resolve(obj); resolve(obj);
}); });
} }

5
src/gender/gender.ts
View File

@ -21,12 +21,11 @@ export async function load(config) {
export async function predict(image, config) { export async function predict(image, config) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.gender.skipFrames) && config.videoOptimized && last.gender !== '') { if ((skipped < config.face.gender.skipFrames) && config.skipFrame && last.gender !== '') {
skipped++; skipped++;
return last; return last;
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
let enhance; let enhance;

7
src/handpose/handpipeline.ts
View File

@ -81,11 +81,12 @@ export class HandPipeline {
// run new detector every skipFrames unless we only want box to start with // run new detector every skipFrames unless we only want box to start with
let boxes; let boxes;
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.videoOptimized) {
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image, config); boxes = await this.handDetector.estimateHandBounds(image, config);
this.skipped = 0; this.skipped = 0;
} }
if (config.videoOptimized) this.skipped++; if (config.skipFrame) this.skipped++;
// if detector result count doesn't match current working set, use it to reset current working set // if detector result count doesn't match current working set, use it to reset current working set
if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) { if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) {
@ -96,8 +97,6 @@ export class HandPipeline {
} }
const hands: Array<{}> = []; const hands: Array<{}> = [];
if (config.hand.skipInitial && this.detectedHands === 0) this.skipped = 0;
// go through working set of boxes // go through working set of boxes
for (let i = 0; i < this.storedBoxes.length; i++) { for (let i = 0; i < this.storedBoxes.length; i++) {
const currentBox = this.storedBoxes[i]; const currentBox = this.storedBoxes[i];

43
src/human.ts
View File

@ -4,7 +4,7 @@ import { Result } from './result';
import * as sysinfo from './sysinfo'; import * as sysinfo from './sysinfo';
import * as tf from '../dist/tfjs.esm.js'; import * as tf from '../dist/tfjs.esm.js';
import * as backend from './tfjs/backend'; import * as backend from './tfjs/backend';
import * as faceall from './faceall'; import * as face from './face';
import * as facemesh from './blazeface/facemesh'; import * as facemesh from './blazeface/facemesh';
import * as faceres from './faceres/faceres'; import * as faceres from './faceres/faceres';
import * as emotion from './emotion/emotion'; import * as emotion from './emotion/emotion';
@ -116,6 +116,7 @@ export class Human {
#analyzeMemoryLeaks: boolean; #analyzeMemoryLeaks: boolean;
#checkSanity: boolean; #checkSanity: boolean;
#firstRun: boolean; #firstRun: boolean;
#lastInputSum: number
// definition end // definition end
@ -165,6 +166,7 @@ export class Human {
this.faceUVMap = facemesh.uvmap; this.faceUVMap = facemesh.uvmap;
// include platform info // include platform info
this.sysinfo = sysinfo.info(); this.sysinfo = sysinfo.info();
this.#lastInputSum = 1;
} }
// helper function: measure tensor leak // helper function: measure tensor leak
@ -338,6 +340,21 @@ export class Human {
} }
} }
// check if input changed sufficiently to trigger new detections
/** @hidden */
#skipFrame = async (input) => {
if (this.config.cacheSensitivity === 0) return true;
const resizeFact = 32;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced);
reduced.dispose();
const sum = sumT.dataSync()[0] as number;
sumT.dispose();
const diff = Math.max(sum, this.#lastInputSum) / Math.min(sum, this.#lastInputSum) - 1;
this.#lastInputSum = sum;
return diff < this.config.cacheSensitivity;
}
/** Main detection method /** Main detection method
* - Analyze configuration: {@link Config} * - Analyze configuration: {@link Config}
* - Pre-process input: {@link Input} * - Pre-process input: {@link Input}
@ -369,6 +386,8 @@ export class Human {
// load models if enabled // load models if enabled
await this.load(); await this.load();
/*
// function disabled in favor of inputChanged
// disable video optimization for inputs of type image, but skip if inside worker thread // disable video optimization for inputs of type image, but skip if inside worker thread
let previousVideoOptimized; let previousVideoOptimized;
// @ts-ignore ignore missing type for WorkerGlobalScope as that is the point // @ts-ignore ignore missing type for WorkerGlobalScope as that is the point
@ -382,6 +401,7 @@ export class Human {
previousVideoOptimized = this.config.videoOptimized; previousVideoOptimized = this.config.videoOptimized;
this.config.videoOptimized = false; this.config.videoOptimized = false;
} }
*/
timeStamp = now(); timeStamp = now();
const process = image.process(input, this.config); const process = image.process(input, this.config);
@ -393,6 +413,17 @@ export class Human {
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze('Get Image:'); this.analyze('Get Image:');
timeStamp = now();
// @ts-ignore hidden dynamic property that is not part of definitions
this.config.skipFrame = await this.#skipFrame(process.tensor);
if (!this.perf.frames) this.perf.frames = 0;
if (!this.perf.cached) this.perf.cached = 0;
this.perf.frames++;
// @ts-ignore hidden dynamic property that is not part of definitions
if (this.config.skipFrame) this.perf.cached++;
this.perf.changed = Math.trunc(now() - timeStamp);
this.analyze('Check Changed:');
// prepare where to store model results // prepare where to store model results
let bodyRes; let bodyRes;
let handRes; let handRes;
@ -402,12 +433,12 @@ export class Human {
// run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
if (this.config.async) { if (this.config.async) {
faceRes = this.config.face.enabled ? faceall.detectFace(this, process.tensor) : []; faceRes = this.config.face.enabled ? face.detectFace(this, process.tensor) : [];
if (this.perf.face) delete this.perf.face; if (this.perf.face) delete this.perf.face;
} else { } else {
this.state = 'run:face'; this.state = 'run:face';
timeStamp = now(); timeStamp = now();
faceRes = this.config.face.enabled ? await faceall.detectFace(this, process.tensor) : []; faceRes = this.config.face.enabled ? await face.detectFace(this, process.tensor) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) this.perf.face = current; if (current > 0) this.perf.face = current;
} }
@ -471,9 +502,6 @@ export class Human {
else if (this.perf.gesture) delete this.perf.gesture; else if (this.perf.gesture) delete this.perf.gesture;
} }
// restore video optimizations if previously disabled
if (previousVideoOptimized) this.config.videoOptimized = previousVideoOptimized;
this.perf.total = Math.trunc(now() - timeStart); this.perf.total = Math.trunc(now() - timeStart);
this.state = 'idle'; this.state = 'idle';
const result = { const result = {
@ -577,13 +605,10 @@ export class Human {
const t0 = now(); const t0 = now();
if (userConfig) this.config = mergeDeep(this.config, userConfig); if (userConfig) this.config = mergeDeep(this.config, userConfig);
if (!this.config.warmup || this.config.warmup === 'none') return { error: 'null' }; if (!this.config.warmup || this.config.warmup === 'none') return { error: 'null' };
const save = this.config.videoOptimized;
this.config.videoOptimized = false;
let res; let res;
if (typeof createImageBitmap === 'function') res = await this.#warmupBitmap(); if (typeof createImageBitmap === 'function') res = await this.#warmupBitmap();
else if (typeof Image !== 'undefined') res = await this.#warmupCanvas(); else if (typeof Image !== 'undefined') res = await this.#warmupCanvas();
else res = await this.#warmupNode(); else res = await this.#warmupNode();
this.config.videoOptimized = save;
const t1 = now(); const t1 = now();
if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res); if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res);
return res; return res;

6
src/nanodet/nanodet.ts
View File

@ -97,13 +97,11 @@ async function process(res, inputSize, outputShape, config) {
export async function predict(image, config) { export async function predict(image, config) {
if (!model) return null; if (!model) return null;
// console.log(skipped, config.object.skipFrames, config.videoOptimized, ((skipped < config.object.skipFrames) && config.videoOptimized && (last.length > 0))); if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
if ((skipped < config.object.skipFrames) && config.videoOptimized && (last.length > 0)) {
skipped++; skipped++;
return last; return last;
} }
if (config.videoOptimized) skipped = 0; skipped = 0;
else skipped = Number.MAX_SAFE_INTEGER;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image.shape[2], image.shape[1]]; const outputSize = [image.shape[2], image.shape[1]];
const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false); const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false);

1
test/test-node-gpu.js
View File

@ -5,7 +5,6 @@ const config = {
modelBasePath: 'file://models/', modelBasePath: 'file://models/',
backend: 'tensorflow', backend: 'tensorflow',
debug: false, debug: false,
videoOptimized: false,
async: false, async: false,
filter: { filter: {
enabled: true, enabled: true,

1
test/test-node-wasm.js
View File

@ -6,7 +6,6 @@ const config = {
backend: 'wasm', backend: 'wasm',
wasmPath: 'node_modules/@tensorflow/tfjs-backend-wasm/dist/', wasmPath: 'node_modules/@tensorflow/tfjs-backend-wasm/dist/',
debug: false, debug: false,
videoOptimized: false,
async: false, async: false,
filter: { filter: {
enabled: true, enabled: true,

1
test/test-node.js
View File

@ -5,7 +5,6 @@ const config = {
modelBasePath: 'file://models/', modelBasePath: 'file://models/',
backend: 'tensorflow', backend: 'tensorflow',
debug: false, debug: false,
videoOptimized: false,
async: false, async: false,
filter: { filter: {
enabled: true, enabled: true,