ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

complete async work

pull/193/head
Vladimir Mandic 2021-08-14 11:16:26 -04:00
parent 26b3fa28cf
commit 38714b023f
19 changed files with 146403 additions and 3673 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
demo/index.js
View File

@ -41,9 +41,9 @@ let userConfig = {
flip: false, flip: false,
}, },
face: { enabled: true, face: { enabled: true,
detector: { return: false }, detector: { return: false, rotation: true },
mesh: { enabled: true }, mesh: { enabled: true },
iris: { enabled: false }, iris: { enabled: true },
description: { enabled: false }, description: { enabled: false },
emotion: { enabled: false }, emotion: { enabled: false },
}, },
@ -441,9 +441,9 @@ function webWorker(input, image, canvas, timestamp) {
// main processing function when input is webcam, can use direct invocation or web worker // main processing function when input is webcam, can use direct invocation or web worker
function runHumanDetect(input, canvas, timestamp) { function runHumanDetect(input, canvas, timestamp) {
// if live video // if live video
const videoLive = (input.readyState > 2) && (!input.paused); const videoLive = input.readyState > 2;
const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live'); const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live';
const live = videoLive || cameraLive; const live = (videoLive || cameraLive) && (!input.paused);
if (!live) { if (!live) {
// stop ui refresh // stop ui refresh
// if (ui.drawThread) cancelAnimationFrame(ui.drawThread); // if (ui.drawThread) cancelAnimationFrame(ui.drawThread);

11055
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

69032
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

69060
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

213
dist/human.node-gpu.js vendored
View File

@ -512,7 +512,7 @@ var BlazeFaceModel = class {
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits));
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
@ -520,8 +520,9 @@ var BlazeFaceModel = class {
const nms = await nmsTensor.array(); const nms = await nmsTensor.array();
tf3.dispose(nmsTensor); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
const scoresData = await scores.data();
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scoresData[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
@ -533,6 +534,7 @@ var BlazeFaceModel = class {
} }
tf3.dispose(batch); tf3.dispose(batch);
tf3.dispose(boxes); tf3.dispose(boxes);
tf3.dispose(scores);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3910,7 +3912,9 @@ var Pipeline = class {
box6.endPoint[0] / this.meshSize box6.endPoint[0] / this.meshSize
]], [0], [this.irisSize, this.irisSize]); ]], [0], [this.irisSize, this.irisSize]);
if (flip && tf4.ENV.flags.IS_BROWSER) { if (flip && tf4.ENV.flags.IS_BROWSER) {
crop = tf4.image.flipLeftRight(crop); const flipped = tf4.image.flipLeftRight(crop);
tf4.dispose(crop);
crop = flipped;
} }
return { box: box6, boxSize, crop }; return { box: box6, boxSize, crop };
} }
@ -3942,6 +3946,47 @@ var Pipeline = class {
return [coord[0], coord[1], z]; return [coord[0], coord[1], z];
}); });
} }
correctFaceRotation(config3, box6, input) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
const angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
const rotationMatrix = buildRotationMatrix(-angle, faceCenter);
const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]);
const face5 = tf4.div(cut, 255);
tf4.dispose(cut);
tf4.dispose(rotatedImage);
return [angle, rotationMatrix, face5];
}
async augmentIris(rawCoords, face5) {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const combined = tf4.concat([leftEyeCrop, rightEyeCrop]);
tf4.dispose(leftEyeCrop);
tf4.dispose(rightEyeCrop);
const eyePredictions = this.irisModel.predict(combined);
tf4.dispose(combined);
const eyePredictionsData = await eyePredictions.data();
tf4.dispose(eyePredictions);
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]);
}
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
const newCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
return newCoords;
}
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
@ -3985,93 +4030,65 @@ var Pipeline = class {
tf4.dispose(prediction.landmarks); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = [];
for (let i = 0; i < this.storedBoxes.length; i++) {
let box6 = this.storedBoxes[i];
let face5; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine; [angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled)
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255); face5 = tf4.div(cut, 255);
else tf4.dispose(cut);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255); tf4.dispose(clonedImage);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { results.push({
mesh: [], mesh: [],
box: box6, box: box6,
faceConfidence: null, faceConfidence: null,
boxConfidence: box6.confidence, boxConfidence: box6.confidence,
confidence: box6.confidence, confidence: box6.confidence,
image: face5 image: face5
}; });
return prediction2; } else {
} const [contours, confidence, contourCoords] = this.meshDetector.execute(face5);
const [, confidence, contourCoords] = this.meshDetector.execute(face5); tf4.dispose(contours);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = (await confidence.data())[0];
if (faceConfidence < config3.face.detector.minConfidence) { tf4.dispose(confidence);
this.storedBoxes[i].confidence = faceConfidence; const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
return null; let rawCoords = await coordsReshaped.array();
} tf4.dispose(contourCoords);
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); tf4.dispose(coordsReshaped);
let rawCoords = coordsReshaped.arraySync(); if (faceConfidence < config3.face.detector.minConfidence) {
if (config3.face.iris.enabled) { this.storedBoxes[i].confidence = faceConfidence;
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); tf4.dispose(face5);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else { } else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]); if (config3.face.iris.enabled)
rawCoords = await this.augmentIris(rawCoords, face5);
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix);
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
[angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
}
results.push({
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
confidence: faceConfidence,
image: face5
});
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
} }
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
} }
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix); }
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
}
const prediction = {
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
image: face5
};
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
return prediction;
}));
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence); this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
this.detectedFaces = results.length; this.detectedFaces = results.length;
@ -4239,19 +4256,19 @@ async function predict2(image18, config3, idx, count2) {
resT = await model.predict(enhanced); resT = await model.predict(enhanced);
tf6.dispose(enhanced); tf6.dispose(enhanced);
if (resT) { if (resT) {
tf6.tidy(() => { const gender = await resT.find((t) => t.shape[1] === 1).data();
const gender = resT.find((t) => t.shape[1] === 1).dataSync(); const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100;
const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100; if (confidence > config3.face.description.minConfidence) {
if (confidence > config3.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.genderScore = Math.min(0.99, confidence);
obj.genderScore = Math.min(0.99, confidence); }
} const argmax = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1);
const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; const age = (await argmax.data())[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = await resT.find((t) => t.shape[1] === 100).data();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
obj.descriptor = [...desc.dataSync()]; const descriptor = await desc.data();
}); obj.descriptor = [...descriptor];
resT.forEach((t) => tf6.dispose(t)); resT.forEach((t) => tf6.dispose(t));
} }
last[idx] = obj; last[idx] = obj;
@ -7909,7 +7926,7 @@ var HandPipeline = class {
tf11.dispose(rotatedImage); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
@ -8303,8 +8320,8 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast5 = tf15.cast(resize, "int32"); const cast4 = tf15.cast(resize, "int32");
return cast5; return cast4;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
@ -11375,10 +11392,24 @@ var Human = class {
if (this.config.backend && this.config.backend.length > 0) { if (this.config.backend && this.config.backend.length > 0) {
if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug) if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug)
log("running inside web worker"); log("running inside web worker");
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") {
this.config.backend = "webgl"; if (this.config.debug)
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) log("override: backend set to tensorflow while running in browser");
this.config.backend = "humangl";
}
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) {
if (this.config.debug)
log("override: backend set to webgl while running in nodejs");
this.config.backend = "tensorflow"; this.config.backend = "tensorflow";
}
const available = Object.keys(this.tf.engine().registryFactory);
if (this.config.debug)
log("available backends:", available);
if (!available.includes(this.config.backend)) {
log(`error: backend ${this.config.backend} not found in registry`);
this.config.backend = this.tf.ENV.flags.IS_NODE ? "tensorflow" : "humangl";
log(`override: using backend ${this.config.backend} instead`);
}
if (this.config.debug) if (this.config.debug)
log("setting backend:", this.config.backend); log("setting backend:", this.config.backend);
if (this.config.backend === "wasm") { if (this.config.backend === "wasm") {

213
dist/human.node-wasm.js vendored
View File

@ -513,7 +513,7 @@ var BlazeFaceModel = class {
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits));
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
@ -521,8 +521,9 @@ var BlazeFaceModel = class {
const nms = await nmsTensor.array(); const nms = await nmsTensor.array();
tf3.dispose(nmsTensor); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
const scoresData = await scores.data();
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scoresData[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
@ -534,6 +535,7 @@ var BlazeFaceModel = class {
} }
tf3.dispose(batch); tf3.dispose(batch);
tf3.dispose(boxes); tf3.dispose(boxes);
tf3.dispose(scores);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3911,7 +3913,9 @@ var Pipeline = class {
box6.endPoint[0] / this.meshSize box6.endPoint[0] / this.meshSize
]], [0], [this.irisSize, this.irisSize]); ]], [0], [this.irisSize, this.irisSize]);
if (flip && tf4.ENV.flags.IS_BROWSER) { if (flip && tf4.ENV.flags.IS_BROWSER) {
crop = tf4.image.flipLeftRight(crop); const flipped = tf4.image.flipLeftRight(crop);
tf4.dispose(crop);
crop = flipped;
} }
return { box: box6, boxSize, crop }; return { box: box6, boxSize, crop };
} }
@ -3943,6 +3947,47 @@ var Pipeline = class {
return [coord[0], coord[1], z]; return [coord[0], coord[1], z];
}); });
} }
correctFaceRotation(config3, box6, input) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
const angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
const rotationMatrix = buildRotationMatrix(-angle, faceCenter);
const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]);
const face5 = tf4.div(cut, 255);
tf4.dispose(cut);
tf4.dispose(rotatedImage);
return [angle, rotationMatrix, face5];
}
async augmentIris(rawCoords, face5) {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const combined = tf4.concat([leftEyeCrop, rightEyeCrop]);
tf4.dispose(leftEyeCrop);
tf4.dispose(rightEyeCrop);
const eyePredictions = this.irisModel.predict(combined);
tf4.dispose(combined);
const eyePredictionsData = await eyePredictions.data();
tf4.dispose(eyePredictions);
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]);
}
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
const newCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
return newCoords;
}
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
@ -3986,93 +4031,65 @@ var Pipeline = class {
tf4.dispose(prediction.landmarks); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = [];
for (let i = 0; i < this.storedBoxes.length; i++) {
let box6 = this.storedBoxes[i];
let face5; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine; [angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled)
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255); face5 = tf4.div(cut, 255);
else tf4.dispose(cut);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255); tf4.dispose(clonedImage);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { results.push({
mesh: [], mesh: [],
box: box6, box: box6,
faceConfidence: null, faceConfidence: null,
boxConfidence: box6.confidence, boxConfidence: box6.confidence,
confidence: box6.confidence, confidence: box6.confidence,
image: face5 image: face5
}; });
return prediction2; } else {
} const [contours, confidence, contourCoords] = this.meshDetector.execute(face5);
const [, confidence, contourCoords] = this.meshDetector.execute(face5); tf4.dispose(contours);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = (await confidence.data())[0];
if (faceConfidence < config3.face.detector.minConfidence) { tf4.dispose(confidence);
this.storedBoxes[i].confidence = faceConfidence; const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
return null; let rawCoords = await coordsReshaped.array();
} tf4.dispose(contourCoords);
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); tf4.dispose(coordsReshaped);
let rawCoords = coordsReshaped.arraySync(); if (faceConfidence < config3.face.detector.minConfidence) {
if (config3.face.iris.enabled) { this.storedBoxes[i].confidence = faceConfidence;
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); tf4.dispose(face5);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else { } else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]); if (config3.face.iris.enabled)
rawCoords = await this.augmentIris(rawCoords, face5);
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix);
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
[angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
}
results.push({
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
confidence: faceConfidence,
image: face5
});
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
} }
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
} }
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix); }
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
}
const prediction = {
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
image: face5
};
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
return prediction;
}));
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence); this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
this.detectedFaces = results.length; this.detectedFaces = results.length;
@ -4240,19 +4257,19 @@ async function predict2(image18, config3, idx, count2) {
resT = await model.predict(enhanced); resT = await model.predict(enhanced);
tf6.dispose(enhanced); tf6.dispose(enhanced);
if (resT) { if (resT) {
tf6.tidy(() => { const gender = await resT.find((t) => t.shape[1] === 1).data();
const gender = resT.find((t) => t.shape[1] === 1).dataSync(); const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100;
const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100; if (confidence > config3.face.description.minConfidence) {
if (confidence > config3.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.genderScore = Math.min(0.99, confidence);
obj.genderScore = Math.min(0.99, confidence); }
} const argmax = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1);
const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; const age = (await argmax.data())[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = await resT.find((t) => t.shape[1] === 100).data();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
obj.descriptor = [...desc.dataSync()]; const descriptor = await desc.data();
}); obj.descriptor = [...descriptor];
resT.forEach((t) => tf6.dispose(t)); resT.forEach((t) => tf6.dispose(t));
} }
last[idx] = obj; last[idx] = obj;
@ -7910,7 +7927,7 @@ var HandPipeline = class {
tf11.dispose(rotatedImage); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
@ -8304,8 +8321,8 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast5 = tf15.cast(resize, "int32"); const cast4 = tf15.cast(resize, "int32");
return cast5; return cast4;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
@ -11376,10 +11393,24 @@ var Human = class {
if (this.config.backend && this.config.backend.length > 0) { if (this.config.backend && this.config.backend.length > 0) {
if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug) if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug)
log("running inside web worker"); log("running inside web worker");
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") {
this.config.backend = "webgl"; if (this.config.debug)
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) log("override: backend set to tensorflow while running in browser");
this.config.backend = "humangl";
}
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) {
if (this.config.debug)
log("override: backend set to webgl while running in nodejs");
this.config.backend = "tensorflow"; this.config.backend = "tensorflow";
}
const available = Object.keys(this.tf.engine().registryFactory);
if (this.config.debug)
log("available backends:", available);
if (!available.includes(this.config.backend)) {
log(`error: backend ${this.config.backend} not found in registry`);
this.config.backend = this.tf.ENV.flags.IS_NODE ? "tensorflow" : "humangl";
log(`override: using backend ${this.config.backend} instead`);
}
if (this.config.debug) if (this.config.debug)
log("setting backend:", this.config.backend); log("setting backend:", this.config.backend);
if (this.config.backend === "wasm") { if (this.config.backend === "wasm") {

213
dist/human.node.js vendored
View File

@ -512,7 +512,7 @@ var BlazeFaceModel = class {
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf3.slice(batchOut, [0, 0], [-1, 1]); const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf3.squeeze(tf3.sigmoid(logits)).dataSync(); const scoresOut = tf3.squeeze(tf3.sigmoid(logits));
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
this.config = mergeDeep(this.config, userConfig); this.config = mergeDeep(this.config, userConfig);
@ -520,8 +520,9 @@ var BlazeFaceModel = class {
const nms = await nmsTensor.array(); const nms = await nmsTensor.array();
tf3.dispose(nmsTensor); tf3.dispose(nmsTensor);
const annotatedBoxes = []; const annotatedBoxes = [];
const scoresData = await scores.data();
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scoresData[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = createBox(boundingBox); const localBox = createBox(boundingBox);
@ -533,6 +534,7 @@ var BlazeFaceModel = class {
} }
tf3.dispose(batch); tf3.dispose(batch);
tf3.dispose(boxes); tf3.dispose(boxes);
tf3.dispose(scores);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize] scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@ -3910,7 +3912,9 @@ var Pipeline = class {
box6.endPoint[0] / this.meshSize box6.endPoint[0] / this.meshSize
]], [0], [this.irisSize, this.irisSize]); ]], [0], [this.irisSize, this.irisSize]);
if (flip && tf4.ENV.flags.IS_BROWSER) { if (flip && tf4.ENV.flags.IS_BROWSER) {
crop = tf4.image.flipLeftRight(crop); const flipped = tf4.image.flipLeftRight(crop);
tf4.dispose(crop);
crop = flipped;
} }
return { box: box6, boxSize, crop }; return { box: box6, boxSize, crop };
} }
@ -3942,6 +3946,47 @@ var Pipeline = class {
return [coord[0], coord[1], z]; return [coord[0], coord[1], z];
}); });
} }
correctFaceRotation(config3, box6, input) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
const angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
const rotationMatrix = buildRotationMatrix(-angle, faceCenter);
const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]);
const face5 = tf4.div(cut, 255);
tf4.dispose(cut);
tf4.dispose(rotatedImage);
return [angle, rotationMatrix, face5];
}
async augmentIris(rawCoords, face5) {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const combined = tf4.concat([leftEyeCrop, rightEyeCrop]);
tf4.dispose(leftEyeCrop);
tf4.dispose(rightEyeCrop);
const eyePredictions = this.irisModel.predict(combined);
tf4.dispose(combined);
const eyePredictionsData = await eyePredictions.data();
tf4.dispose(eyePredictions);
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]);
}
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
const newCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
return newCoords;
}
async predict(input, config3) { async predict(input, config3) {
let useFreshBox = false; let useFreshBox = false;
let detector; let detector;
@ -3985,93 +4030,65 @@ var Pipeline = class {
tf4.dispose(prediction.landmarks); tf4.dispose(prediction.landmarks);
}); });
} }
const results = tf4.tidy(() => this.storedBoxes.map((box6, i) => { const results = [];
for (let i = 0; i < this.storedBoxes.length; i++) {
let box6 = this.storedBoxes[i];
let face5; let face5;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) { if (config3.face.detector.rotation && config3.face.mesh.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine; [angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(input, angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
if (config3.face.mesh.enabled)
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = IDENTITY_MATRIX; rotationMatrix = IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config3.face.mesh.enabled) const cut = config3.face.mesh.enabled ? cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]) : cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255); face5 = tf4.div(cut, 255);
else tf4.dispose(cut);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255); tf4.dispose(clonedImage);
} }
if (!config3.face.mesh.enabled) { if (!config3.face.mesh.enabled) {
const prediction2 = { results.push({
mesh: [], mesh: [],
box: box6, box: box6,
faceConfidence: null, faceConfidence: null,
boxConfidence: box6.confidence, boxConfidence: box6.confidence,
confidence: box6.confidence, confidence: box6.confidence,
image: face5 image: face5
}; });
return prediction2; } else {
} const [contours, confidence, contourCoords] = this.meshDetector.execute(face5);
const [, confidence, contourCoords] = this.meshDetector.execute(face5); tf4.dispose(contours);
const faceConfidence = confidence.dataSync()[0]; const faceConfidence = (await confidence.data())[0];
if (faceConfidence < config3.face.detector.minConfidence) { tf4.dispose(confidence);
this.storedBoxes[i].confidence = faceConfidence; const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
return null; let rawCoords = await coordsReshaped.array();
} tf4.dispose(contourCoords);
const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]); tf4.dispose(coordsReshaped);
let rawCoords = coordsReshaped.arraySync(); if (faceConfidence < config3.face.detector.minConfidence) {
if (config3.face.iris.enabled) { this.storedBoxes[i].confidence = faceConfidence;
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); tf4.dispose(face5);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face5, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf4.concat([leftEyeCrop, rightEyeCrop]));
const eyePredictionsData = eyePredictions.dataSync();
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", null);
} else if (leftToRightEyeDepthDifference < 1) {
replaceRawCoordinates(rawCoords, leftEyeRawCoords, "left", ["EyeUpper0", "EyeLower0"]);
} else { } else {
replaceRawCoordinates(rawCoords, rightEyeRawCoords, "right", ["EyeUpper0", "EyeLower0"]); if (config3.face.iris.enabled)
rawCoords = await this.augmentIris(rawCoords, face5);
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix);
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
[angle, rotationMatrix, face5] = this.correctFaceRotation(config3, box6, input);
}
results.push({
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
confidence: faceConfidence,
image: face5
});
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
} }
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, "left");
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
} }
const mesh = this.transformRawCoords(rawCoords, box6, angle, rotationMatrix); }
const storeConfidence = box6.confidence;
box6 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
box6.confidence = storeConfidence;
if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = box6.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
angle = computeRotation(box6.landmarks[indexOfMouth], box6.landmarks[indexOfForehead]);
const faceCenter = getBoxCenter({ startPoint: box6.startPoint, endPoint: box6.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf4.image.rotateWithOffset(tf4.cast(input, "float32"), angle, 0, faceCenterNormalized);
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face5 = tf4.div(cutBoxFromImageAndResize({ startPoint: box6.startPoint, endPoint: box6.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
}
const prediction = {
mesh,
box: box6,
faceConfidence,
boxConfidence: box6.confidence,
image: face5
};
this.storedBoxes[i] = { ...squarifyBox(box6), confidence: box6.confidence, faceConfidence };
return prediction;
}));
if (config3.face.mesh.enabled) if (config3.face.mesh.enabled)
this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence); this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
this.detectedFaces = results.length; this.detectedFaces = results.length;
@ -4239,19 +4256,19 @@ async function predict2(image18, config3, idx, count2) {
resT = await model.predict(enhanced); resT = await model.predict(enhanced);
tf6.dispose(enhanced); tf6.dispose(enhanced);
if (resT) { if (resT) {
tf6.tidy(() => { const gender = await resT.find((t) => t.shape[1] === 1).data();
const gender = resT.find((t) => t.shape[1] === 1).dataSync(); const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100;
const confidence = Math.trunc(200 * Math.abs(gender[0] - 0.5)) / 100; if (confidence > config3.face.description.minConfidence) {
if (confidence > config3.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? "female" : "male";
obj.gender = gender[0] <= 0.5 ? "female" : "male"; obj.genderScore = Math.min(0.99, confidence);
obj.genderScore = Math.min(0.99, confidence); }
} const argmax = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1);
const age = tf6.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; const age = (await argmax.data())[0];
const all2 = resT.find((t) => t.shape[1] === 100).dataSync(); const all2 = await resT.find((t) => t.shape[1] === 100).data();
obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10; obj.age = Math.round(all2[age - 1] > all2[age + 1] ? 10 * age - 100 * all2[age - 1] : 10 * age + 100 * all2[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
obj.descriptor = [...desc.dataSync()]; const descriptor = await desc.data();
}); obj.descriptor = [...descriptor];
resT.forEach((t) => tf6.dispose(t)); resT.forEach((t) => tf6.dispose(t));
} }
last[idx] = obj; last[idx] = obj;
@ -7909,7 +7926,7 @@ var HandPipeline = class {
tf11.dispose(rotatedImage); tf11.dispose(rotatedImage);
const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage); const [confidenceT, keypoints3] = await this.handPoseModel.predict(handImage);
tf11.dispose(handImage); tf11.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = (await confidenceT.data())[0];
tf11.dispose(confidenceT); tf11.dispose(confidenceT);
if (confidence >= config3.hand.minConfidence) { if (confidence >= config3.hand.minConfidence) {
const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]); const keypointsReshaped = tf11.reshape(keypoints3, [-1, 3]);
@ -8303,8 +8320,8 @@ async function predict8(image18, config3) {
if (!model6.inputs[0].shape) if (!model6.inputs[0].shape)
return null; return null;
const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false); const resize = tf15.image.resizeBilinear(image18, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
const cast5 = tf15.cast(resize, "int32"); const cast4 = tf15.cast(resize, "int32");
return cast5; return cast4;
}); });
let resT; let resT;
if (config3.body.enabled) if (config3.body.enabled)
@ -11375,10 +11392,24 @@ var Human = class {
if (this.config.backend && this.config.backend.length > 0) { if (this.config.backend && this.config.backend.length > 0) {
if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug) if (typeof window === "undefined" && typeof WorkerGlobalScope !== "undefined" && this.config.debug)
log("running inside web worker"); log("running inside web worker");
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === "tensorflow") {
this.config.backend = "webgl"; if (this.config.debug)
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) log("override: backend set to tensorflow while running in browser");
this.config.backend = "humangl";
}
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === "webgl" || this.config.backend === "humangl")) {
if (this.config.debug)
log("override: backend set to webgl while running in nodejs");
this.config.backend = "tensorflow"; this.config.backend = "tensorflow";
}
const available = Object.keys(this.tf.engine().registryFactory);
if (this.config.debug)
log("available backends:", available);
if (!available.includes(this.config.backend)) {
log(`error: backend ${this.config.backend} not found in registry`);
this.config.backend = this.tf.ENV.flags.IS_NODE ? "tensorflow" : "humangl";
log(`override: using backend ${this.config.backend} instead`);
}
if (this.config.debug) if (this.config.debug)
log("setting backend:", this.config.backend); log("setting backend:", this.config.backend);
if (this.config.backend === "wasm") { if (this.config.backend === "wasm") {

2
package.json
View File

@ -83,7 +83,7 @@
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"seedrandom": "^3.0.5", "seedrandom": "^3.0.5",
"simple-git": "^2.43.0", "simple-git": "^2.44.0",
"tslib": "^2.3.1", "tslib": "^2.3.1",
"typedoc": "0.21.5", "typedoc": "0.21.5",
"typescript": "4.3.5" "typescript": "4.3.5"

6
src/blazeface/blazeface.ts
View File

@ -57,7 +57,7 @@ export class BlazeFaceModel {
} }
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]); const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf.slice(batchOut, [0, 0], [-1, 1]); const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf.squeeze(tf.sigmoid(logits)).dataSync(); // inside tf.tidy const scoresOut = tf.squeeze(tf.sigmoid(logits)); // inside tf.tidy
return [batchOut, boxesOut, scoresOut]; return [batchOut, boxesOut, scoresOut];
}); });
@ -67,8 +67,9 @@ export class BlazeFaceModel {
const nms = await nmsTensor.array(); const nms = await nmsTensor.array();
tf.dispose(nmsTensor); tf.dispose(nmsTensor);
const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: number[], confidence: number }> = []; const annotatedBoxes: Array<{ box: { startPoint: Tensor, endPoint: Tensor }, landmarks: Tensor, anchor: number[], confidence: number }> = [];
const scoresData = await scores.data();
for (let i = 0; i < nms.length; i++) { for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]]; const confidence = scoresData[nms[i]];
if (confidence > this.config.face.detector.minConfidence) { if (confidence > this.config.face.detector.minConfidence) {
const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]); const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
const localBox = box.createBox(boundingBox); const localBox = box.createBox(boundingBox);
@ -80,6 +81,7 @@ export class BlazeFaceModel {
} }
tf.dispose(batch); tf.dispose(batch);
tf.dispose(boxes); tf.dispose(boxes);
tf.dispose(scores);
return { return {
boxes: annotatedBoxes, boxes: annotatedBoxes,
scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize], scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize],

179
src/blazeface/facepipeline.ts
View File

@ -115,7 +115,9 @@ export class Pipeline {
box.endPoint[0] / this.meshSize, box.endPoint[0] / this.meshSize,
]], [0], [this.irisSize, this.irisSize]); ]], [0], [this.irisSize, this.irisSize]);
if (flip && tf.ENV.flags.IS_BROWSER) { if (flip && tf.ENV.flags.IS_BROWSER) {
crop = tf.image.flipLeftRight(crop); // flipLeftRight is not defined for tfjs-node const flipped = tf.image.flipLeftRight(crop); // flipLeftRight is not defined for tfjs-node
tf.dispose(crop);
crop = flipped;
} }
return { box, boxSize, crop }; return { box, boxSize, crop };
} }
@ -153,6 +155,53 @@ export class Pipeline {
}); });
} }
correctFaceRotation(config, box, input) {
const [indexOfMouth, indexOfForehead] = (box.landmarks.length >= meshLandmarks.count) ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
const angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
const rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
const cut = config.face.mesh.enabled
? bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize])
: bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.boxSize, this.boxSize]);
const face = tf.div(cut, 255);
tf.dispose(cut);
tf.dispose(rotatedImage);
return [angle, rotationMatrix, face];
}
async augmentIris(rawCoords, face) {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const combined = tf.concat([leftEyeCrop, rightEyeCrop]);
tf.dispose(leftEyeCrop);
tf.dispose(rightEyeCrop);
const eyePredictions = this.irisModel.predict(combined) as Tensor;
tf.dispose(combined);
const eyePredictionsData = await eyePredictions.data(); // inside tf.tidy
tf.dispose(eyePredictions);
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize);
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords);
if (Math.abs(leftToRightEyeDepthDifference) < 30) { // User is looking straight ahead.
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', null);
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', null);
// If the user is looking to the left or to the right, the iris coordinates tend to diverge too much from the mesh coordinates for them to be merged
// So we only update a single contour line above and below the eye.
} else if (leftToRightEyeDepthDifference < 1) { // User is looking towards the right.
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', ['EyeUpper0', 'EyeLower0']);
} else { // User is looking towards the left.
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', ['EyeUpper0', 'EyeLower0']);
}
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, 'left');
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, 'right');
const newCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
return newCoords;
}
async predict(input, config) { async predict(input, config) {
let useFreshBox = false; let useFreshBox = false;
// run new detector every skipFrames unless we only want box to start with // run new detector every skipFrames unless we only want box to start with
@ -198,106 +247,78 @@ export class Pipeline {
tf.dispose(prediction.landmarks); tf.dispose(prediction.landmarks);
}); });
} }
const results = tf.tidy(() => this.storedBoxes.map((box, i) => {
// The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box). const results: Array<{ mesh, box, faceConfidence, boxConfidence, confidence, image }> = [];
for (let i = 0; i < this.storedBoxes.length; i++) {
let box = this.storedBoxes[i]; // The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
let face; let face;
let angle = 0; let angle = 0;
let rotationMatrix; let rotationMatrix;
if (config.face.detector.rotation && config.face.mesh.enabled && tf.ENV.flags.IS_BROWSER) { if (config.face.detector.rotation && config.face.mesh.enabled && tf.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = (box.landmarks.length >= meshLandmarks.count) ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine; [angle, rotationMatrix, face] = this.correctFaceRotation(config, box, input);
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
if (config.face.mesh.enabled) face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
else face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.boxSize, this.boxSize]), 255);
} else { } else {
rotationMatrix = util.IDENTITY_MATRIX; rotationMatrix = util.IDENTITY_MATRIX;
const clonedImage = input.clone(); const clonedImage = input.clone();
if (config.face.mesh.enabled) face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.meshSize, this.meshSize]), 255); const cut = config.face.mesh.enabled
else face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.boxSize, this.boxSize]), 255); ? bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.meshSize, this.meshSize])
: bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, clonedImage, [this.boxSize, this.boxSize]);
face = tf.div(cut, 255);
tf.dispose(cut);
tf.dispose(clonedImage);
} }
// if we're not going to produce mesh, don't spend time with further processing // if we're not going to produce mesh, don't spend time with further processing
if (!config.face.mesh.enabled) { if (!config.face.mesh.enabled) {
const prediction = { results.push({
mesh: [], mesh: [],
box, box,
faceConfidence: null, faceConfidence: null,
boxConfidence: box.confidence, boxConfidence: box.confidence,
confidence: box.confidence, confidence: box.confidence,
image: face, image: face,
}; });
return prediction; } else {
} const [contours, confidence, contourCoords] = this.meshDetector.execute(face) as Array<Tensor>; // The first returned tensor represents facial contours which are already included in the coordinates.
tf.dispose(contours);
const faceConfidence = (await confidence.data())[0] as number; // inside tf.tidy
tf.dispose(confidence);
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
let rawCoords = await coordsReshaped.array();
tf.dispose(contourCoords);
tf.dispose(coordsReshaped);
if (faceConfidence < config.face.detector.minConfidence) {
this.storedBoxes[i].confidence = faceConfidence; // reset confidence of cached box
tf.dispose(face);
} else {
if (config.face.iris.enabled) rawCoords = await this.augmentIris(rawCoords, face);
const [, confidence, contourCoords] = this.meshDetector.execute(face) as Array<Tensor>; // The first returned tensor represents facial contours which are already included in the coordinates. // override box from detection with one calculated from mesh
const faceConfidence = confidence.dataSync()[0] as number; // inside tf.tidy const mesh = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
if (faceConfidence < config.face.detector.minConfidence) { const storeConfidence = box.confidence;
this.storedBoxes[i].confidence = faceConfidence; // reset confidence of cached box // @ts-ignore enlargeBox does not include confidence so we append it manually
return null; // if below confidence just exit box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(mesh), 1.5); // redefine box with mesh calculated one
} box.confidence = storeConfidence;
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
let rawCoords = coordsReshaped.arraySync();
if (config.face.iris.enabled) { // do rotation one more time with mesh keypoints if we want to return perfect image
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true); if (config.face.detector.rotation && config.face.mesh.enabled && config.face.description.enabled && tf.ENV.flags.IS_BROWSER) {
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]); [angle, rotationMatrix, face] = this.correctFaceRotation(config, box, input);
const eyePredictions = this.irisModel.predict(tf.concat([leftEyeCrop, rightEyeCrop])) as Tensor; }
const eyePredictionsData = eyePredictions.dataSync(); // inside tf.tidy
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3); results.push({
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true); mesh,
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3); box,
const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = this.getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize); faceConfidence,
const leftToRightEyeDepthDifference = this.getLeftToRightEyeDepthDifference(rawCoords); boxConfidence: box.confidence,
if (Math.abs(leftToRightEyeDepthDifference) < 30) { // User is looking straight ahead. confidence: faceConfidence,
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', null); image: face,
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', null); });
// If the user is looking to the left or to the right, the iris coordinates tend to diverge too much from the mesh coordinates for them to be merged
// So we only update a single contour line above and below the eye. // updated stored cache values
} else if (leftToRightEyeDepthDifference < 1) { // User is looking towards the right. this.storedBoxes[i] = { ...bounding.squarifyBox(box), confidence: box.confidence, faceConfidence };
replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', ['EyeUpper0', 'EyeLower0']);
} else { // User is looking towards the left.
replaceRawCoordinates(rawCoords, rightEyeRawCoords, 'right', ['EyeUpper0', 'EyeLower0']);
} }
const adjustedLeftIrisCoords = this.getAdjustedIrisCoords(rawCoords, leftIrisRawCoords, 'left');
const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, 'right');
rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
} }
}
// override box from detection with one calculated from mesh
const mesh = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
const storeConfidence = box.confidence;
// @ts-ignore enlargeBox does not include confidence so we append it manually
box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(mesh), 1.5); // redefine box with mesh calculated one
box.confidence = storeConfidence;
// do rotation one more time with mesh keypoints if we want to return perfect image
if (config.face.detector.rotation && config.face.mesh.enabled && config.face.description.enabled && tf.ENV.flags.IS_BROWSER) {
const [indexOfMouth, indexOfForehead] = (box.landmarks.length >= meshLandmarks.count) ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
angle = util.computeRotation(box.landmarks[indexOfMouth], box.landmarks[indexOfForehead]);
const faceCenter = bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(tf.cast(input, 'float32'), angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = util.buildRotationMatrix(-angle, faceCenter);
face = tf.div(bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshSize, this.meshSize]), 255);
}
const prediction = {
mesh,
box,
faceConfidence,
boxConfidence: box.confidence,
image: face,
};
// updated stored cache values
this.storedBoxes[i] = { ...bounding.squarifyBox(box), confidence: box.confidence, faceConfidence };
return prediction;
}));
// results = results.filter((a) => a !== null); // results = results.filter((a) => a !== null);
// remove cache entries for detected boxes on low confidence // remove cache entries for detected boxes on low confidence

12
src/efficientpose/efficientpose.ts
View File

@ -34,14 +34,10 @@ export async function load(config: Config): Promise<GraphModel> {
function max2d(inputs, minScore) { function max2d(inputs, minScore) {
const [width, height] = inputs.shape; const [width, height] = inputs.shape;
return tf.tidy(() => { return tf.tidy(() => {
// modulus op implemented in tf const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); // modulus op implemented in tf
const mod = (a, b) => tf.sub(a, tf.mul(tf.div(a, tf.scalar(b, 'int32')), tf.scalar(b, 'int32'))); const reshaped = tf.reshape(inputs, [height * width]); // combine all data
// combine all data const newScore = tf.max(reshaped, 0).dataSync()[0]; // get highest score // inside tf.tidy
const reshaped = tf.reshape(inputs, [height * width]); if (newScore > minScore) { // skip coordinate calculation is score is too low
// get highest score
const newScore = tf.max(reshaped, 0).dataSync()[0]; // inside tf.tidy
if (newScore > minScore) {
// skip coordinate calculation is score is too low
const coords = tf.argMax(reshaped, 0); const coords = tf.argMax(reshaped, 0);
const x = mod(coords, width).dataSync()[0]; // inside tf.tidy const x = mod(coords, width).dataSync()[0]; // inside tf.tidy
const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy

8
src/embedding/embedding.ts
View File

@ -95,7 +95,7 @@ export async function predict(input, config): Promise<number[]> {
let data: Array<number> = []; let data: Array<number> = [];
if (config.face.embedding.enabled) { if (config.face.embedding.enabled) {
const image = enhance(input); const image = enhance(input);
data = tf.tidy(() => { const dataT = tf.tidy(() => {
/* /*
// if needed convert from NHWC to NCHW // if needed convert from NHWC to NCHW
const nchw = image.transpose([3, 0, 1, 2]); const nchw = image.transpose([3, 0, 1, 2]);
@ -125,9 +125,11 @@ export async function predict(input, config): Promise<number[]> {
const reshape = tf.reshape(res, [128, 2]); // split 256 vectors into 128 x 2 const reshape = tf.reshape(res, [128, 2]); // split 256 vectors into 128 x 2
const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it
const output: Array<number> = reduce.dataSync(); // inside tf.tidy return reduce;
return [...output]; // convert typed array to simple array
}); });
const output: Array<number> = await dataT.data();
data = [...output]; // convert typed array to simple array
tf.dispose(dataT);
tf.dispose(image); tf.dispose(image);
} }
resolve(data); resolve(data);

30
src/faceres/faceres.ts
View File

@ -133,23 +133,23 @@ export async function predict(image: Tensor, config: Config, idx, count) {
tf.dispose(enhanced); tf.dispose(enhanced);
if (resT) { if (resT) {
tf.tidy(() => { const gender = await resT.find((t) => t.shape[1] === 1).data();
const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; if (confidence > config.face.description.minConfidence) {
if (confidence > config.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.genderScore = Math.min(0.99, confidence);
obj.genderScore = Math.min(0.99, confidence); }
} const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1);
const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy const age = (await argmax.data())[0];
const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy const all = await resT.find((t) => t.shape[1] === 100).data(); // inside tf.tidy
obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
// const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8 // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
// const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
obj.descriptor = [...desc.dataSync()]; // inside tf.tidy const descriptor = await desc.data();
}); obj.descriptor = [...descriptor];
resT.forEach((t) => tf.dispose(t)); resT.forEach((t) => tf.dispose(t));
} }

6
src/gear/agegenderrace.ts
View File

@ -67,14 +67,14 @@ export async function predict(image: Tensor, config: Config) {
// tf.dispose(enhance); // tf.dispose(enhance);
if (ageT) { if (ageT) {
// const data = ageT.dataSync(); // const data = await ageT.data();
// {0: 'below_20', 1: '21-25', 2: '26-30', 3: '31-40',4: '41-50', 5: '51-60', 6: 'Above60'} // {0: 'below_20', 1: '21-25', 2: '26-30', 3: '31-40',4: '41-50', 5: '51-60', 6: 'Above60'}
} }
if (genderT) { if (genderT) {
// const data = genderT.dataSync(); // const data = await genderT.data();
} }
if (raceT) { if (raceT) {
// const data = raceT.dataSync(); // const data = await raceT.data();
// {0: 'white', 1: 'black', 2: 'asian', 3: 'indian', 4: 'others'} // {0: 'white', 1: 'black', 2: 'asian', 3: 'indian', 4: 'others'}
} }

6
src/gender/gender.ts
View File

@ -87,10 +87,10 @@ export async function predict(image: Tensor, config: Config | any) {
obj.confidence = Math.min(0.99, confidence); obj.confidence = Math.min(0.99, confidence);
} }
/* /*
let age = genderT[1].argMax(1).dataSync()[0]; let age = (await genderT[1].argMax(1).data())[0];
const all = genderT[1].dataSync(); const all = await genderT[1].data();
age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
const descriptor = genderT[1].dataSync(); const descriptor = await genderT[1].data();
*/ */
genderT.forEach((t) => tf.dispose(t)); genderT.forEach((t) => tf.dispose(t));
} }

2
src/handpose/handpipeline.ts
View File

@ -118,7 +118,7 @@ export class HandPipeline {
tf.dispose(rotatedImage); tf.dispose(rotatedImage);
const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>; const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>;
tf.dispose(handImage); tf.dispose(handImage);
const confidence = confidenceT.dataSync()[0]; const confidence = (await confidenceT.data())[0];
tf.dispose(confidenceT); tf.dispose(confidenceT);
if (confidence >= config.hand.minConfidence) { if (confidence >= config.hand.minConfidence) {
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]); const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);

21
src/human.ts
View File

@ -302,8 +302,23 @@ export class Human {
if (typeof window === 'undefined' && typeof WorkerGlobalScope !== 'undefined' && this.config.debug) log('running inside web worker'); if (typeof window === 'undefined' && typeof WorkerGlobalScope !== 'undefined' && this.config.debug) log('running inside web worker');
// force browser vs node backend // force browser vs node backend
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl'; if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') {
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) this.config.backend = 'tensorflow'; if (this.config.debug) log('override: backend set to tensorflow while running in browser');
this.config.backend = 'humangl';
}
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'humangl')) {
if (this.config.debug) log('override: backend set to webgl while running in nodejs');
this.config.backend = 'tensorflow';
}
const available = Object.keys(this.tf.engine().registryFactory);
if (this.config.debug) log('available backends:', available);
if (!available.includes(this.config.backend)) {
log(`error: backend ${this.config.backend} not found in registry`);
this.config.backend = this.tf.ENV.flags.IS_NODE ? 'tensorflow' : 'humangl';
log(`override: using backend ${this.config.backend} instead`);
}
if (this.config.debug) log('setting backend:', this.config.backend); if (this.config.debug) log('setting backend:', this.config.backend);
@ -363,7 +378,7 @@ export class Human {
// use tensor sum // use tensor sum
/* /*
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
const sum = sumT.dataSync()[0] as number; const sum = await sumT.data()[0] as number;
sumT.dispose(); sumT.dispose();
*/ */
// use js loop sum, faster than uploading tensor to gpu calculating and downloading back // use js loop sum, faster than uploading tensor to gpu calculating and downloading back