ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

major performance improvements for all models

pull/50/head
Vladimir Mandic 2020-11-02 18:54:03 -05:00
parent bdb22d3f15
commit 7b15efd028
23 changed files with 886 additions and 734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config.js
View File

@ -4,6 +4,9 @@
export default {
backend: 'webgl', // select tfjs backend to use
console: true, // enable debugging output to console
async: false, // execute enabled models in parallel
// this disables per-model performance data but slightly increases performance
// cannot be used if profiling is enabled
profile: false, // enable tfjs profiling
// this has significant performance impact, only enable for debugging purposes
// currently only implemented for age,gender,emotion models

10
demo/browser.js
View File

@ -90,14 +90,18 @@ const log = (...msg) => {
// draws processed results and starts processing of a next frame
function drawResults(input, result, canvas) {
// update fps
// update fps data
fps.push(1000 / (performance.now() - timeStamp));
if (fps.length > ui.maxFrames) fps.shift();
menu.updateChart('FPS', fps);
// enable for continous performance monitoring
// console.log(result.performance);
// eslint-disable-next-line no-use-before-define
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop before we even draw results
// draw fps chart
menu.updateChart('FPS', fps);
// draw image from video
const ctx = canvas.getContext('2d');
ctx.fillStyle = ui.baseBackground;

244
dist/human.esm-nobundle.js vendored
View File

@ -109,14 +109,14 @@ var require_blazeface = __commonJS((exports) => {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -143,9 +143,12 @@ var require_blazeface = __commonJS((exports) => {
}
async estimateFaces(input) {
const {boxes, scaleFactor} = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData.map((landmark) => [
@ -162,8 +165,9 @@ var require_blazeface = __commonJS((exports) => {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}
async function load(config) {
@ -3916,7 +3920,7 @@ var require_ssrnet = __commonJS((exports) => {
let ageT;
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled)
promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled)
@ -3933,12 +3937,12 @@ var require_ssrnet = __commonJS((exports) => {
profile2.run("gender", profileGender);
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf2.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? "female" : "male";
@ -3993,11 +3997,11 @@ var require_emotion = __commonJS((exports) => {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf2.dispose(emotionT);
} else {
const profileData = await tf2.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile2.run("emotion", profileData);
}
@ -4580,10 +4584,7 @@ var require_box2 = __commonJS((exports) => {
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return {startPoint, endPoint, palmLandmarks};
}
exports.scaleBoxCoordinates = scaleBoxCoordinates;
@ -4657,8 +4658,7 @@ var require_handdetector = __commonJS((exports) => {
const rawBoxes = tf2.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf2.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -4670,7 +4670,7 @@ var require_handdetector = __commonJS((exports) => {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
async estimateHandBounds(input, config) {
@ -4678,12 +4678,10 @@ var require_handdetector = __commonJS((exports) => {
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || predictions.length === 0)
@ -4691,10 +4689,10 @@ var require_handdetector = __commonJS((exports) => {
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({startPoint, endPoint, palmLandmarks}, [input.shape[2] / this.width, input.shape[1] / this.height]));
@ -4802,11 +4800,11 @@ var require_pipeline2 = __commonJS((exports) => {
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -4872,7 +4870,7 @@ var require_pipeline2 = __commonJS((exports) => {
if (!this.regionsOfInterest)
return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox)
return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
@ -4885,18 +4883,18 @@ var require_pipeline2 = __commonJS((exports) => {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf2.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -4904,7 +4902,7 @@ var require_pipeline2 = __commonJS((exports) => {
this.updateRegionsOfInterest(nextBoundingBox, false, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint
@ -5729,6 +5727,7 @@ var require_config = __commonJS((exports) => {
var config_default = {
backend: "webgl",
console: true,
async: false,
profile: false,
deallocate: false,
scoped: false,
@ -5947,10 +5946,12 @@ class Human {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas() : null;
this.fx = null;
this.state = "idle";
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
this.inCanvas = null;
this.outCanvas = null;
this.models = {
facemesh: null,
posenet: null,
@ -6027,12 +6028,14 @@ class Human {
}
}
tfImage(input) {
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0)
targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0)
@ -6041,60 +6044,69 @@ class Human {
targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0)
targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (offscreenCanvas.width !== targetWidth)
offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight)
offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext("2d");
if (!this.inCanvas || this.inCanvas.width !== originalWidth || this.inCanvas.height !== originalHeight) {
this.inCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (this.inCanvas.width !== targetWidth)
this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight)
this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext("2d");
if (input instanceof ImageData)
ctx.putImageData(input, 0, 0);
else
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || this.inCanvas.width !== this.outCanvas.width || this.inCanvas.height !== this.outCanvas.height) {
this.outCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement("canvas");
if (this.outCanvas.width !== this.inCanvas.width)
this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height)
this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx)
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas({canvas: this.outCanvas}) : null;
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas)
this.outCanvas = this.inCanvas;
let pixels;
if (this.config.backend === "webgl" || canvas instanceof ImageData) {
pixels = tf.browser.fromPixels(canvas);
if (this.config.backend === "webgl" || this.outCanvas instanceof ImageData) {
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d");
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -6103,7 +6115,7 @@ class Human {
pixels.dispose();
casted.dispose();
}
return {tensor, canvas: this.config.filter.return ? filtered : null};
return {tensor, canvas: this.config.filter.return ? this.outCanvas : null};
}
async detect(input, userConfig = {}) {
this.state = "config";
@ -6119,6 +6131,10 @@ class Human {
return {error};
}
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
timeStamp = now();
await this.checkBackend();
@ -6140,18 +6156,28 @@ class Human {
const image = this.tfImage(input);
perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
}
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async)
[poseRes, handRes] = await Promise.all([poseRes, handRes]);
const faceRes = [];
if (this.config.face.enabled) {
this.state = "run:face";
@ -6166,11 +6192,11 @@ class Human {
}
this.state = "run:agegender";
timeStamp = now();
const ssrData = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
this.state = "run:emotion";
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -6179,10 +6205,10 @@ class Human {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
});
this.analyze("End FaceMesh:");

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

36
dist/human.esm-nobundle.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"config.js": {
"bytes": 7075,
"bytes": 7322,
"imports": []
},
"package.json": {
@ -9,7 +9,7 @@
"imports": []
},
"src/emotion/emotion.js": {
"bytes": 1958,
"bytes": 1954,
"imports": [
{
"path": "src/profile.js"
@ -17,7 +17,7 @@
]
},
"src/facemesh/blazeface.js": {
"bytes": 7161,
"bytes": 6991,
"imports": []
},
"src/facemesh/box.js": {
@ -75,11 +75,11 @@
"imports": []
},
"src/handpose/box.js": {
"bytes": 2627,
"bytes": 2572,
"imports": []
},
"src/handpose/handdetector.js": {
"bytes": 4248,
"bytes": 4077,
"imports": [
{
"path": "src/handpose/box.js"
@ -105,7 +105,7 @@
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8169,
"bytes": 8216,
"imports": [
{
"path": "src/handpose/box.js"
@ -120,7 +120,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13768,
"bytes": 15047,
"imports": [
{
"path": "src/facemesh/facemesh.js"
@ -263,7 +263,7 @@
"imports": []
},
"src/ssrnet/ssrnet.js": {
"bytes": 2115,
"bytes": 2127,
"imports": [
{
"path": "src/profile.js"
@ -275,13 +275,13 @@
"dist/human.esm-nobundle.js.map": {
"imports": [],
"inputs": {},
"bytes": 254548
"bytes": 256450
},
"dist/human.esm-nobundle.js": {
"imports": [],
"inputs": {
"src/facemesh/blazeface.js": {
"bytesInOutput": 7121
"bytesInOutput": 7133
},
"src/facemesh/keypoints.js": {
"bytesInOutput": 2768
@ -308,10 +308,10 @@
"bytesInOutput": 1092
},
"src/ssrnet/ssrnet.js": {
"bytesInOutput": 2310
"bytesInOutput": 2322
},
"src/emotion/emotion.js": {
"bytesInOutput": 2044
"bytesInOutput": 2040
},
"src/posenet/modelBase.js": {
"bytesInOutput": 910
@ -347,10 +347,10 @@
"bytesInOutput": 903
},
"src/handpose/box.js": {
"bytesInOutput": 2805
"bytesInOutput": 2744
},
"src/handpose/handdetector.js": {
"bytesInOutput": 4117
"bytesInOutput": 4027
},
"src/handpose/keypoints.js": {
"bytesInOutput": 263
@ -359,7 +359,7 @@
"bytesInOutput": 2663
},
"src/handpose/pipeline.js": {
"bytesInOutput": 7604
"bytesInOutput": 7651
},
"src/handpose/handpose.js": {
"bytesInOutput": 2270
@ -368,19 +368,19 @@
"bytesInOutput": 20137
},
"config.js": {
"bytesInOutput": 2273
"bytesInOutput": 2291
},
"package.json": {
"bytesInOutput": 3012
},
"src/human.js": {
"bytesInOutput": 11904
"bytesInOutput": 13241
},
"src/human.js": {
"bytesInOutput": 0
}
},
"bytes": 158147
"bytes": 159418
}
}
}

244
dist/human.esm.js vendored
View File

@ -67158,14 +67158,14 @@ var require_blazeface = __commonJS((exports) => {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -67192,9 +67192,12 @@ var require_blazeface = __commonJS((exports) => {
}
async estimateFaces(input) {
const {boxes, scaleFactor} = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData.map((landmark) => [
@ -67211,8 +67214,9 @@ var require_blazeface = __commonJS((exports) => {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}
async function load(config) {
@ -70965,7 +70969,7 @@ var require_ssrnet = __commonJS((exports) => {
let ageT;
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled)
promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled)
@ -70982,12 +70986,12 @@ var require_ssrnet = __commonJS((exports) => {
profile2.run("gender", profileGender);
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf2.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? "female" : "male";
@ -71042,11 +71046,11 @@ var require_emotion = __commonJS((exports) => {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf2.dispose(emotionT);
} else {
const profileData = await tf2.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile2.run("emotion", profileData);
}
@ -71629,10 +71633,7 @@ var require_box2 = __commonJS((exports) => {
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return {startPoint, endPoint, palmLandmarks};
}
exports.scaleBoxCoordinates = scaleBoxCoordinates;
@ -71706,8 +71707,7 @@ var require_handdetector = __commonJS((exports) => {
const rawBoxes = tf2.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf2.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -71719,7 +71719,7 @@ var require_handdetector = __commonJS((exports) => {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
async estimateHandBounds(input, config) {
@ -71727,12 +71727,10 @@ var require_handdetector = __commonJS((exports) => {
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || predictions.length === 0)
@ -71740,10 +71738,10 @@ var require_handdetector = __commonJS((exports) => {
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({startPoint, endPoint, palmLandmarks}, [input.shape[2] / this.width, input.shape[1] / this.height]));
@ -71851,11 +71849,11 @@ var require_pipeline2 = __commonJS((exports) => {
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -71921,7 +71919,7 @@ var require_pipeline2 = __commonJS((exports) => {
if (!this.regionsOfInterest)
return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox)
return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
@ -71934,18 +71932,18 @@ var require_pipeline2 = __commonJS((exports) => {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf2.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -71953,7 +71951,7 @@ var require_pipeline2 = __commonJS((exports) => {
this.updateRegionsOfInterest(nextBoundingBox, false, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint
@ -72778,6 +72776,7 @@ var require_config = __commonJS((exports) => {
var config_default = {
backend: "webgl",
console: true,
async: false,
profile: false,
deallocate: false,
scoped: false,
@ -72996,10 +72995,12 @@ class Human {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas() : null;
this.fx = null;
this.state = "idle";
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
this.inCanvas = null;
this.outCanvas = null;
this.models = {
facemesh: null,
posenet: null,
@ -73076,12 +73077,14 @@ class Human {
}
}
tfImage(input) {
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0)
targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0)
@ -73090,60 +73093,69 @@ class Human {
targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0)
targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (offscreenCanvas.width !== targetWidth)
offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight)
offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext("2d");
if (!this.inCanvas || this.inCanvas.width !== originalWidth || this.inCanvas.height !== originalHeight) {
this.inCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (this.inCanvas.width !== targetWidth)
this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight)
this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext("2d");
if (input instanceof ImageData)
ctx.putImageData(input, 0, 0);
else
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || this.inCanvas.width !== this.outCanvas.width || this.inCanvas.height !== this.outCanvas.height) {
this.outCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement("canvas");
if (this.outCanvas.width !== this.inCanvas.width)
this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height)
this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx)
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas({canvas: this.outCanvas}) : null;
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas)
this.outCanvas = this.inCanvas;
let pixels;
if (this.config.backend === "webgl" || canvas instanceof ImageData) {
pixels = tf.browser.fromPixels(canvas);
if (this.config.backend === "webgl" || this.outCanvas instanceof ImageData) {
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d");
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -73152,7 +73164,7 @@ class Human {
pixels.dispose();
casted.dispose();
}
return {tensor, canvas: this.config.filter.return ? filtered : null};
return {tensor, canvas: this.config.filter.return ? this.outCanvas : null};
}
async detect(input, userConfig = {}) {
this.state = "config";
@ -73168,6 +73180,10 @@ class Human {
return {error};
}
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
timeStamp = now();
await this.checkBackend();
@ -73189,18 +73205,28 @@ class Human {
const image = this.tfImage(input);
perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
}
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async)
[poseRes, handRes] = await Promise.all([poseRes, handRes]);
const faceRes = [];
if (this.config.face.enabled) {
this.state = "run:face";
@ -73215,11 +73241,11 @@ class Human {
}
this.state = "run:agegender";
timeStamp = now();
const ssrData = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
this.state = "run:emotion";
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -73228,10 +73254,10 @@ class Human {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
});
this.analyze("End FaceMesh:");

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

36
dist/human.esm.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"config.js": {
"bytes": 7075,
"bytes": 7322,
"imports": []
},
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
@ -153,7 +153,7 @@
"imports": []
},
"src/emotion/emotion.js": {
"bytes": 1958,
"bytes": 1954,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -164,7 +164,7 @@
]
},
"src/facemesh/blazeface.js": {
"bytes": 7161,
"bytes": 6991,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -236,7 +236,7 @@
"imports": []
},
"src/handpose/box.js": {
"bytes": 2627,
"bytes": 2572,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -244,7 +244,7 @@
]
},
"src/handpose/handdetector.js": {
"bytes": 4248,
"bytes": 4077,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -276,7 +276,7 @@
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8169,
"bytes": 8216,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -294,7 +294,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13768,
"bytes": 15047,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -450,7 +450,7 @@
"imports": []
},
"src/ssrnet/ssrnet.js": {
"bytes": 2115,
"bytes": 2127,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -481,7 +481,7 @@
"dist/human.esm.js.map": {
"imports": [],
"inputs": {},
"bytes": 5127854
"bytes": 5129756
},
"dist/human.esm.js": {
"imports": [],
@ -544,7 +544,7 @@
"bytesInOutput": 3025
},
"src/facemesh/blazeface.js": {
"bytesInOutput": 7111
"bytesInOutput": 7123
},
"src/facemesh/keypoints.js": {
"bytesInOutput": 2768
@ -571,10 +571,10 @@
"bytesInOutput": 1092
},
"src/ssrnet/ssrnet.js": {
"bytesInOutput": 2300
"bytesInOutput": 2312
},
"src/emotion/emotion.js": {
"bytesInOutput": 2034
"bytesInOutput": 2030
},
"src/posenet/modelBase.js": {
"bytesInOutput": 900
@ -610,10 +610,10 @@
"bytesInOutput": 903
},
"src/handpose/box.js": {
"bytesInOutput": 2795
"bytesInOutput": 2734
},
"src/handpose/handdetector.js": {
"bytesInOutput": 4107
"bytesInOutput": 4017
},
"src/handpose/keypoints.js": {
"bytesInOutput": 263
@ -622,7 +622,7 @@
"bytesInOutput": 2663
},
"src/handpose/pipeline.js": {
"bytesInOutput": 7594
"bytesInOutput": 7641
},
"src/handpose/handpose.js": {
"bytesInOutput": 2260
@ -631,19 +631,19 @@
"bytesInOutput": 20137
},
"config.js": {
"bytesInOutput": 2273
"bytesInOutput": 2291
},
"package.json": {
"bytesInOutput": 3012
},
"src/human.js": {
"bytesInOutput": 11894
"bytesInOutput": 13231
},
"src/human.js": {
"bytesInOutput": 0
}
},
"bytes": 2927113
"bytes": 2928384
}
}
}

244
dist/human.js vendored
View File

@ -67159,14 +67159,14 @@ var Human = (() => {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -67193,9 +67193,12 @@ var Human = (() => {
}
async estimateFaces(input) {
const {boxes, scaleFactor} = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData.map((landmark) => [
@ -67212,8 +67215,9 @@ var Human = (() => {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}
async function load(config) {
@ -70966,7 +70970,7 @@ var Human = (() => {
let ageT;
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled)
promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled)
@ -70983,12 +70987,12 @@ var Human = (() => {
profile.run("gender", profileGender);
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? "female" : "male";
@ -71043,11 +71047,11 @@ var Human = (() => {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf.dispose(emotionT);
} else {
const profileData = await tf.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile.run("emotion", profileData);
}
@ -71630,10 +71634,7 @@ var Human = (() => {
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return {startPoint, endPoint, palmLandmarks};
}
exports.scaleBoxCoordinates = scaleBoxCoordinates;
@ -71707,8 +71708,7 @@ var Human = (() => {
const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -71720,7 +71720,7 @@ var Human = (() => {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
async estimateHandBounds(input, config) {
@ -71728,12 +71728,10 @@ var Human = (() => {
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || predictions.length === 0)
@ -71741,10 +71739,10 @@ var Human = (() => {
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({startPoint, endPoint, palmLandmarks}, [input.shape[2] / this.width, input.shape[1] / this.height]));
@ -71852,11 +71850,11 @@ var Human = (() => {
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -71922,7 +71920,7 @@ var Human = (() => {
if (!this.regionsOfInterest)
return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox)
return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
@ -71935,18 +71933,18 @@ var Human = (() => {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -71954,7 +71952,7 @@ var Human = (() => {
this.updateRegionsOfInterest(nextBoundingBox, false, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint
@ -72779,6 +72777,7 @@ var Human = (() => {
var config_default = {
backend: "webgl",
console: true,
async: false,
profile: false,
deallocate: false,
scoped: false,
@ -73001,10 +73000,12 @@ var Human = (() => {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas() : null;
this.fx = null;
this.state = "idle";
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
this.inCanvas = null;
this.outCanvas = null;
this.models = {
facemesh: null,
posenet: null,
@ -73081,12 +73082,14 @@ var Human = (() => {
}
}
tfImage(input) {
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0)
targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0)
@ -73095,60 +73098,69 @@ var Human = (() => {
targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0)
targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (offscreenCanvas.width !== targetWidth)
offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight)
offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext("2d");
if (!this.inCanvas || this.inCanvas.width !== originalWidth || this.inCanvas.height !== originalHeight) {
this.inCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (this.inCanvas.width !== targetWidth)
this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight)
this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext("2d");
if (input instanceof ImageData)
ctx.putImageData(input, 0, 0);
else
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || this.inCanvas.width !== this.outCanvas.width || this.inCanvas.height !== this.outCanvas.height) {
this.outCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement("canvas");
if (this.outCanvas.width !== this.inCanvas.width)
this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height)
this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx)
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas({canvas: this.outCanvas}) : null;
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas)
this.outCanvas = this.inCanvas;
let pixels;
if (this.config.backend === "webgl" || canvas instanceof ImageData) {
pixels = tf.browser.fromPixels(canvas);
if (this.config.backend === "webgl" || this.outCanvas instanceof ImageData) {
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d");
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -73157,7 +73169,7 @@ var Human = (() => {
pixels.dispose();
casted.dispose();
}
return {tensor, canvas: this.config.filter.return ? filtered : null};
return {tensor, canvas: this.config.filter.return ? this.outCanvas : null};
}
async detect(input, userConfig = {}) {
this.state = "config";
@ -73173,6 +73185,10 @@ var Human = (() => {
return {error};
}
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
timeStamp = now();
await this.checkBackend();
@ -73194,18 +73210,28 @@ var Human = (() => {
const image = this.tfImage(input);
perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
}
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async)
[poseRes, handRes] = await Promise.all([poseRes, handRes]);
const faceRes = [];
if (this.config.face.enabled) {
this.state = "run:face";
@ -73220,11 +73246,11 @@ var Human = (() => {
}
this.state = "run:agegender";
timeStamp = now();
const ssrData = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
this.state = "run:emotion";
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -73233,10 +73259,10 @@ var Human = (() => {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
});
this.analyze("End FaceMesh:");

4
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

36
dist/human.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"config.js": {
"bytes": 7075,
"bytes": 7322,
"imports": []
},
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
@ -153,7 +153,7 @@
"imports": []
},
"src/emotion/emotion.js": {
"bytes": 1958,
"bytes": 1954,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -164,7 +164,7 @@
]
},
"src/facemesh/blazeface.js": {
"bytes": 7161,
"bytes": 6991,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -236,7 +236,7 @@
"imports": []
},
"src/handpose/box.js": {
"bytes": 2627,
"bytes": 2572,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -244,7 +244,7 @@
]
},
"src/handpose/handdetector.js": {
"bytes": 4248,
"bytes": 4077,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -276,7 +276,7 @@
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8169,
"bytes": 8216,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -294,7 +294,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13768,
"bytes": 15047,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -450,7 +450,7 @@
"imports": []
},
"src/ssrnet/ssrnet.js": {
"bytes": 2115,
"bytes": 2127,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -481,7 +481,7 @@
"dist/human.js.map": {
"imports": [],
"inputs": {},
"bytes": 5131716
"bytes": 5133654
},
"dist/human.js": {
"imports": [],
@ -544,7 +544,7 @@
"bytesInOutput": 3189
},
"src/facemesh/blazeface.js": {
"bytesInOutput": 7400
"bytesInOutput": 7420
},
"src/facemesh/keypoints.js": {
"bytesInOutput": 2936
@ -571,10 +571,10 @@
"bytesInOutput": 1140
},
"src/ssrnet/ssrnet.js": {
"bytesInOutput": 2420
"bytesInOutput": 2432
},
"src/emotion/emotion.js": {
"bytesInOutput": 2137
"bytesInOutput": 2133
},
"src/posenet/modelBase.js": {
"bytesInOutput": 954
@ -610,10 +610,10 @@
"bytesInOutput": 943
},
"src/handpose/box.js": {
"bytesInOutput": 2931
"bytesInOutput": 2864
},
"src/handpose/handdetector.js": {
"bytesInOutput": 4247
"bytesInOutput": 4151
},
"src/handpose/keypoints.js": {
"bytesInOutput": 283
@ -622,7 +622,7 @@
"bytesInOutput": 2803
},
"src/handpose/pipeline.js": {
"bytesInOutput": 7911
"bytesInOutput": 7958
},
"src/handpose/handpose.js": {
"bytesInOutput": 2365
@ -631,16 +631,16 @@
"bytesInOutput": 21557
},
"config.js": {
"bytesInOutput": 2471
"bytesInOutput": 2491
},
"package.json": {
"bytesInOutput": 3144
},
"src/human.js": {
"bytesInOutput": 13235
"bytesInOutput": 14680
}
},
"bytes": 3073187
"bytes": 3074564
}
}
}

244
dist/human.node-nobundle.js vendored
View File

@ -109,14 +109,14 @@ var require_blazeface = __commonJS((exports2) => {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -143,9 +143,12 @@ var require_blazeface = __commonJS((exports2) => {
}
async estimateFaces(input) {
const {boxes, scaleFactor} = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData.map((landmark) => [
@ -162,8 +165,9 @@ var require_blazeface = __commonJS((exports2) => {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}
async function load(config) {
@ -3916,7 +3920,7 @@ var require_ssrnet = __commonJS((exports2) => {
let ageT;
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled)
promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled)
@ -3933,12 +3937,12 @@ var require_ssrnet = __commonJS((exports2) => {
profile2.run("gender", profileGender);
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf2.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? "female" : "male";
@ -3993,11 +3997,11 @@ var require_emotion = __commonJS((exports2) => {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf2.dispose(emotionT);
} else {
const profileData = await tf2.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile2.run("emotion", profileData);
}
@ -4580,10 +4584,7 @@ var require_box2 = __commonJS((exports2) => {
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return {startPoint, endPoint, palmLandmarks};
}
exports2.scaleBoxCoordinates = scaleBoxCoordinates;
@ -4657,8 +4658,7 @@ var require_handdetector = __commonJS((exports2) => {
const rawBoxes = tf2.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf2.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -4670,7 +4670,7 @@ var require_handdetector = __commonJS((exports2) => {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
async estimateHandBounds(input, config) {
@ -4678,12 +4678,10 @@ var require_handdetector = __commonJS((exports2) => {
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || predictions.length === 0)
@ -4691,10 +4689,10 @@ var require_handdetector = __commonJS((exports2) => {
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({startPoint, endPoint, palmLandmarks}, [input.shape[2] / this.width, input.shape[1] / this.height]));
@ -4802,11 +4800,11 @@ var require_pipeline2 = __commonJS((exports2) => {
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -4872,7 +4870,7 @@ var require_pipeline2 = __commonJS((exports2) => {
if (!this.regionsOfInterest)
return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox)
return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
@ -4885,18 +4883,18 @@ var require_pipeline2 = __commonJS((exports2) => {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf2.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -4904,7 +4902,7 @@ var require_pipeline2 = __commonJS((exports2) => {
this.updateRegionsOfInterest(nextBoundingBox, false, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint
@ -5729,6 +5727,7 @@ var require_config = __commonJS((exports2) => {
var config_default = {
backend: "webgl",
console: true,
async: false,
profile: false,
deallocate: false,
scoped: false,
@ -5950,10 +5949,12 @@ class Human {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas() : null;
this.fx = null;
this.state = "idle";
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
this.inCanvas = null;
this.outCanvas = null;
this.models = {
facemesh: null,
posenet: null,
@ -6030,12 +6031,14 @@ class Human {
}
}
tfImage(input) {
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0)
targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0)
@ -6044,60 +6047,69 @@ class Human {
targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0)
targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (offscreenCanvas.width !== targetWidth)
offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight)
offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext("2d");
if (!this.inCanvas || this.inCanvas.width !== originalWidth || this.inCanvas.height !== originalHeight) {
this.inCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (this.inCanvas.width !== targetWidth)
this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight)
this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext("2d");
if (input instanceof ImageData)
ctx.putImageData(input, 0, 0);
else
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || this.inCanvas.width !== this.outCanvas.width || this.inCanvas.height !== this.outCanvas.height) {
this.outCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement("canvas");
if (this.outCanvas.width !== this.inCanvas.width)
this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height)
this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx)
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas({canvas: this.outCanvas}) : null;
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas)
this.outCanvas = this.inCanvas;
let pixels;
if (this.config.backend === "webgl" || canvas instanceof ImageData) {
pixels = tf.browser.fromPixels(canvas);
if (this.config.backend === "webgl" || this.outCanvas instanceof ImageData) {
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d");
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -6106,7 +6118,7 @@ class Human {
pixels.dispose();
casted.dispose();
}
return {tensor, canvas: this.config.filter.return ? filtered : null};
return {tensor, canvas: this.config.filter.return ? this.outCanvas : null};
}
async detect(input, userConfig = {}) {
this.state = "config";
@ -6122,6 +6134,10 @@ class Human {
return {error};
}
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
timeStamp = now();
await this.checkBackend();
@ -6143,18 +6159,28 @@ class Human {
const image = this.tfImage(input);
perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
}
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async)
[poseRes, handRes] = await Promise.all([poseRes, handRes]);
const faceRes = [];
if (this.config.face.enabled) {
this.state = "run:face";
@ -6169,11 +6195,11 @@ class Human {
}
this.state = "run:agegender";
timeStamp = now();
const ssrData = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
this.state = "run:emotion";
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -6182,10 +6208,10 @@ class Human {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
});
this.analyze("End FaceMesh:");

4
dist/human.node-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

244
dist/human.node.js vendored
View File

@ -68200,14 +68200,14 @@ var require_blazeface = __commonJS((exports2) => {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -68234,9 +68234,12 @@ var require_blazeface = __commonJS((exports2) => {
}
async estimateFaces(input) {
const {boxes, scaleFactor} = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData.map((landmark) => [
@ -68253,8 +68256,9 @@ var require_blazeface = __commonJS((exports2) => {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}
async function load(config) {
@ -72007,7 +72011,7 @@ var require_ssrnet = __commonJS((exports2) => {
let ageT;
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled)
promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled)
@ -72024,12 +72028,12 @@ var require_ssrnet = __commonJS((exports2) => {
profile2.run("gender", profileGender);
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf2.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? "female" : "male";
@ -72084,11 +72088,11 @@ var require_emotion = __commonJS((exports2) => {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf2.dispose(emotionT);
} else {
const profileData = await tf2.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile2.run("emotion", profileData);
}
@ -72671,10 +72675,7 @@ var require_box2 = __commonJS((exports2) => {
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return {startPoint, endPoint, palmLandmarks};
}
exports2.scaleBoxCoordinates = scaleBoxCoordinates;
@ -72748,8 +72749,7 @@ var require_handdetector = __commonJS((exports2) => {
const rawBoxes = tf2.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf2.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -72761,7 +72761,7 @@ var require_handdetector = __commonJS((exports2) => {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
async estimateHandBounds(input, config) {
@ -72769,12 +72769,10 @@ var require_handdetector = __commonJS((exports2) => {
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || predictions.length === 0)
@ -72782,10 +72780,10 @@ var require_handdetector = __commonJS((exports2) => {
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({startPoint, endPoint, palmLandmarks}, [input.shape[2] / this.width, input.shape[1] / this.height]));
@ -72893,11 +72891,11 @@ var require_pipeline2 = __commonJS((exports2) => {
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -72963,7 +72961,7 @@ var require_pipeline2 = __commonJS((exports2) => {
if (!this.regionsOfInterest)
return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox)
return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
@ -72976,18 +72974,18 @@ var require_pipeline2 = __commonJS((exports2) => {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf2.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -72995,7 +72993,7 @@ var require_pipeline2 = __commonJS((exports2) => {
this.updateRegionsOfInterest(nextBoundingBox, false, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint
@ -73820,6 +73818,7 @@ var require_config = __commonJS((exports2) => {
var config_default = {
backend: "webgl",
console: true,
async: false,
profile: false,
deallocate: false,
scoped: false,
@ -74041,10 +74040,12 @@ class Human {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas() : null;
this.fx = null;
this.state = "idle";
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
this.inCanvas = null;
this.outCanvas = null;
this.models = {
facemesh: null,
posenet: null,
@ -74121,12 +74122,14 @@ class Human {
}
}
tfImage(input) {
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const originalWidth = input.naturalWidth || input.videoWidth || input.width || input.shape && input.shape[1] > 0;
const originalHeight = input.naturalHeight || input.videoHeight || input.height || input.shape && input.shape[2] > 0;
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0)
targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0)
@ -74135,60 +74138,69 @@ class Human {
targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0)
targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (offscreenCanvas.width !== targetWidth)
offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight)
offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext("2d");
if (!this.inCanvas || this.inCanvas.width !== originalWidth || this.inCanvas.height !== originalHeight) {
this.inCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
if (this.inCanvas.width !== targetWidth)
this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight)
this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext("2d");
if (input instanceof ImageData)
ctx.putImageData(input, 0, 0);
else
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || this.inCanvas.width !== this.outCanvas.width || this.inCanvas.height !== this.outCanvas.height) {
this.outCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement("canvas");
if (this.outCanvas.width !== this.inCanvas.width)
this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height)
this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx)
this.fx = tf.ENV.flags.IS_BROWSER && typeof document !== "undefined" ? new fxImage.Canvas({canvas: this.outCanvas}) : null;
this.fx.reset();
this.fx.addFilter("brightness", this.config.filter.brightness);
if (this.config.filter.contrast !== 0)
this.fx.addFilter("contrast", this.config.filter.contrast);
if (this.config.filter.sharpness !== 0)
this.fx.addFilter("sharpen", this.config.filter.sharpness);
if (this.config.filter.blur !== 0)
this.fx.addFilter("blur", this.config.filter.blur);
if (this.config.filter.saturation !== 0)
this.fx.addFilter("saturation", this.config.filter.saturation);
if (this.config.filter.hue !== 0)
this.fx.addFilter("hue", this.config.filter.hue);
if (this.config.filter.negative)
this.fx.addFilter("negative");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.vintage)
this.fx.addFilter("brownie");
if (this.config.filter.sepia)
this.fx.addFilter("sepia");
if (this.config.filter.kodachrome)
this.fx.addFilter("kodachrome");
if (this.config.filter.technicolor)
this.fx.addFilter("technicolor");
if (this.config.filter.polaroid)
this.fx.addFilter("polaroid");
if (this.config.filter.pixelate !== 0)
this.fx.addFilter("pixelate", this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas)
this.outCanvas = this.inCanvas;
let pixels;
if (this.config.backend === "webgl" || canvas instanceof ImageData) {
pixels = tf.browser.fromPixels(canvas);
if (this.config.backend === "webgl" || this.outCanvas instanceof ImageData) {
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d");
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -74197,7 +74209,7 @@ class Human {
pixels.dispose();
casted.dispose();
}
return {tensor, canvas: this.config.filter.return ? filtered : null};
return {tensor, canvas: this.config.filter.return ? this.outCanvas : null};
}
async detect(input, userConfig = {}) {
this.state = "config";
@ -74213,6 +74225,10 @@ class Human {
return {error};
}
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
timeStamp = now();
await this.checkBackend();
@ -74234,18 +74250,28 @@ class Human {
const image = this.tfImage(input);
perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor;
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = "run:body";
timeStamp = now();
this.analyze("Start PoseNet");
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze("End PoseNet:");
perf.body = Math.trunc(now() - timeStamp);
}
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = "run:hand";
timeStamp = now();
this.analyze("Start HandPose:");
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze("End HandPose:");
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async)
[poseRes, handRes] = await Promise.all([poseRes, handRes]);
const faceRes = [];
if (this.config.face.enabled) {
this.state = "run:face";
@ -74260,11 +74286,11 @@ class Human {
}
this.state = "run:agegender";
timeStamp = now();
const ssrData = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = this.config.face.age.enabled || this.config.face.gender.enabled ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
this.state = "run:emotion";
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
face.image.dispose();
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
@ -74273,10 +74299,10 @@ class Human {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
});
this.analyze("End FaceMesh:");

4
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

36
dist/human.node.json vendored
View File

@ -1,7 +1,7 @@
{
"inputs": {
"config.js": {
"bytes": 7075,
"bytes": 7322,
"imports": []
},
"package.json": {
@ -9,7 +9,7 @@
"imports": []
},
"src/emotion/emotion.js": {
"bytes": 1958,
"bytes": 1954,
"imports": [
{
"path": "src/profile.js"
@ -17,7 +17,7 @@
]
},
"src/facemesh/blazeface.js": {
"bytes": 7161,
"bytes": 6991,
"imports": []
},
"src/facemesh/box.js": {
@ -75,11 +75,11 @@
"imports": []
},
"src/handpose/box.js": {
"bytes": 2627,
"bytes": 2572,
"imports": []
},
"src/handpose/handdetector.js": {
"bytes": 4248,
"bytes": 4077,
"imports": [
{
"path": "src/handpose/box.js"
@ -105,7 +105,7 @@
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8169,
"bytes": 8216,
"imports": [
{
"path": "src/handpose/box.js"
@ -120,7 +120,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13768,
"bytes": 15047,
"imports": [
{
"path": "src/facemesh/facemesh.js"
@ -263,7 +263,7 @@
"imports": []
},
"src/ssrnet/ssrnet.js": {
"bytes": 2115,
"bytes": 2127,
"imports": [
{
"path": "src/profile.js"
@ -275,13 +275,13 @@
"dist/human.node-nobundle.js.map": {
"imports": [],
"inputs": {},
"bytes": 268701
"bytes": 271906
},
"dist/human.node-nobundle.js": {
"imports": [],
"inputs": {
"src/facemesh/blazeface.js": {
"bytesInOutput": 7125
"bytesInOutput": 7137
},
"src/facemesh/keypoints.js": {
"bytesInOutput": 2771
@ -308,10 +308,10 @@
"bytesInOutput": 1095
},
"src/ssrnet/ssrnet.js": {
"bytesInOutput": 2314
"bytesInOutput": 2326
},
"src/emotion/emotion.js": {
"bytesInOutput": 2047
"bytesInOutput": 2043
},
"src/posenet/modelBase.js": {
"bytesInOutput": 912
@ -347,10 +347,10 @@
"bytesInOutput": 917
},
"src/handpose/box.js": {
"bytesInOutput": 2813
"bytesInOutput": 2752
},
"src/handpose/handdetector.js": {
"bytesInOutput": 4119
"bytesInOutput": 4029
},
"src/handpose/keypoints.js": {
"bytesInOutput": 265
@ -359,7 +359,7 @@
"bytesInOutput": 2671
},
"src/handpose/pipeline.js": {
"bytesInOutput": 7606
"bytesInOutput": 7653
},
"src/handpose/handpose.js": {
"bytesInOutput": 2273
@ -368,7 +368,7 @@
"bytesInOutput": 20139
},
"config.js": {
"bytesInOutput": 2275
"bytesInOutput": 2293
},
"package.json": {
"bytesInOutput": 3015
@ -377,10 +377,10 @@
"bytesInOutput": 47
},
"src/human.js": {
"bytesInOutput": 11904
"bytesInOutput": 13241
}
},
"bytes": 158304
"bytes": 159575
}
}
}

4
src/emotion/emotion.js
View File

@ -37,11 +37,11 @@ async function predict(image, config) {
let data;
if (!config.profile) {
const emotionT = await models.emotion.predict(grayscale);
data = await emotionT.data();
data = emotionT.dataSync();
tf.dispose(emotionT);
} else {
const profileData = await tf.profile(() => models.emotion.predict(grayscale));
data = await profileData.result.data();
data = profileData.result.dataSync();
profileData.result.dispose();
profile.run('emotion', profileData);
}

27
src/facemesh/blazeface.js
View File

@ -103,14 +103,15 @@ class BlazeFaceModel {
return [prediction, decodedBounds, scoresOut];
});
const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
const boxIndices = await boxIndicesTensor.array();
const boxIndices = boxIndicesTensor.arraySync();
boxIndicesTensor.dispose();
const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
const vals = await boundingBox.array();
const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
const vals = boundingBox.arraySync();
boundingBox.dispose();
return vals;
}));
});
const annotatedBoxes = [];
for (let i = 0; i < boundingBoxes.length; i++) {
const boundingBox = boundingBoxes[i];
@ -120,12 +121,6 @@ class BlazeFaceModel {
const sliced = tf.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]);
const squeezed = sliced.squeeze();
const landmarks = squeezed.reshape([NUM_LANDMARKS, -1]);
/*
const landmarks = tf
.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1])
.squeeze()
.reshape([NUM_LANDMARKS, -1]);
*/
const probability = tf.slice(scores, [boxIndex], [1]);
const annotatedBox = { box, landmarks, probability, anchor };
annotatedBoxes.push(annotatedBox);
@ -145,9 +140,12 @@ class BlazeFaceModel {
async estimateFaces(input) {
const { boxes, scaleFactor } = await this.getBoundingBoxes(input);
return Promise.all(boxes.map(async (face) => {
const faces = [];
for (const face of boxes) {
const landmarkData = face.landmarks.arraySync();
const scaledBox = scaleBoxFromPrediction(face, scaleFactor);
const [landmarkData, boxData, probabilityData] = await Promise.all([face.landmarks, scaledBox, face.probability].map(async (d) => d.array()));
const boxData = scaleBox.arraySync();
const probabilityData = face.probability.arraySync();
const anchor = face.anchor;
const [scaleFactorX, scaleFactorY] = scaleFactor;
const scaledLandmarks = landmarkData
@ -165,8 +163,9 @@ class BlazeFaceModel {
face.landmarks.dispose();
face.probability.dispose();
scaledBox.dispose();
return normalizedFace;
}));
faces.push(normalizedFace);
}
return faces;
}
}

5
src/handpose/box.js
View File

@ -30,10 +30,7 @@ exports.cutBoxFromImageAndResize = cutBoxFromImageAndResize;
function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]);
return { startPoint, endPoint, palmLandmarks };
}
exports.scaleBoxCoordinates = scaleBoxCoordinates;

21
src/handpose/handdetector.js
View File

@ -40,8 +40,7 @@ class HandDetector {
const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
const boxesWithHands = await boxesWithHandsTensor.array();
const toDispose = [batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf.tidy(() => {
const detectedBoxes = [];
for (const i in boxesWithHands) {
@ -53,7 +52,7 @@ class HandDetector {
}
return detectedBoxes;
});
toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose());
return detectedHands;
}
@ -64,28 +63,24 @@ class HandDetector {
* @param input The image to classify.
*/
async estimateHandBounds(input, config) {
// const inputHeight = input.shape[2];
// const inputWidth = input.shape[1];
this.iouThreshold = config.iouThreshold;
this.scoreThreshold = config.scoreThreshold;
this.maxHands = config.maxHands;
const resized = input.resizeBilinear([this.width, this.height]);
const divided = resized.div(255);
const normalized = divided.sub(0.5);
const image = normalized.mul(2.0);
const divided = resized.mul([1 / 127.5]);
const image = divided.sub(0.5);
resized.dispose();
divided.dispose();
normalized.dispose();
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || (predictions.length === 0)) return null;
const hands = [];
for (const i in predictions) {
const prediction = predictions[i];
const boundingBoxes = await prediction.boxes.array();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = [boundingBoxes[0], boundingBoxes[1]];
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [input.shape[2] / this.width, input.shape[1] / this.height]));

20
src/handpose/pipeline.js
View File

@ -12,11 +12,11 @@ const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
// The Pipeline coordinates between the bounding box and skeleton models.
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, config) {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
@ -93,7 +93,7 @@ class HandPipeline {
const hands = [];
if (!this.regionsOfInterest) return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i][0];
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox) return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
const palmCenter = bounding.getBoxCenter(currentBox);
@ -105,18 +105,18 @@ class HandPipeline {
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [flag, keypoints] = prediction;
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const flagValue = flag.dataSync()[0];
flag.dispose();
if (flagValue < config.minConfidence) {
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = await keypointsReshaped.array();
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
@ -124,7 +124,7 @@ class HandPipeline {
this.updateRegionsOfInterest(nextBoundingBox, false /* force replace */, i);
const result = {
landmarks: coords,
confidence: flagValue,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint,

140
src/human.js
View File

@ -61,10 +61,13 @@ class Human {
this.version = app.version;
this.defaults = defaults;
this.config = defaults;
this.fx = (tf.ENV.flags.IS_BROWSER && (typeof document !== 'undefined')) ? new fxImage.Canvas() : null;
this.fx = null;
this.state = 'idle';
this.numTensors = 0;
this.analyzeMemoryLeaks = false;
// internal temp canvases
this.inCanvas = null;
this.outCanvas = null;
// object that contains all initialized models
this.models = {
facemesh: null,
@ -160,56 +163,62 @@ class Human {
}
tfImage(input) {
// let imageData;
let filtered;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
const originalHeight = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0));
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.fx && this.config.filter.enabled && !(input instanceof tf.Tensor)) {
if (this.config.filter.width > 0) targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0) targetWidth = originalWidth * (this.config.filter.height / originalHeight);
if (this.config.filter.height > 0) targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0) targetHeight = originalHeight * (this.config.filter.width / originalWidth);
const offscreenCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
if (offscreenCanvas.width !== targetWidth) offscreenCanvas.width = targetWidth;
if (offscreenCanvas.height !== targetHeight) offscreenCanvas.height = targetHeight;
const ctx = offscreenCanvas.getContext('2d');
if (input instanceof ImageData) ctx.putImageData(input, 0, 0);
else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
this.fx.reset();
this.fx.addFilter('brightness', this.config.filter.brightness); // must have at least one filter enabled
if (this.config.filter.contrast !== 0) this.fx.addFilter('contrast', this.config.filter.contrast);
if (this.config.filter.sharpness !== 0) this.fx.addFilter('sharpen', this.config.filter.sharpness);
if (this.config.filter.blur !== 0) this.fx.addFilter('blur', this.config.filter.blur);
if (this.config.filter.saturation !== 0) this.fx.addFilter('saturation', this.config.filter.saturation);
if (this.config.filter.hue !== 0) this.fx.addFilter('hue', this.config.filter.hue);
if (this.config.filter.negative) this.fx.addFilter('negative');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.vintage) this.fx.addFilter('brownie');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.kodachrome) this.fx.addFilter('kodachrome');
if (this.config.filter.technicolor) this.fx.addFilter('technicolor');
if (this.config.filter.polaroid) this.fx.addFilter('polaroid');
if (this.config.filter.pixelate !== 0) this.fx.addFilter('pixelate', this.config.filter.pixelate);
filtered = this.fx.apply(offscreenCanvas);
}
let tensor;
if (input instanceof tf.Tensor) {
tensor = tf.clone(input);
} else {
const canvas = filtered || input;
const originalWidth = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
const originalHeight = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0));
let targetWidth = originalWidth;
let targetHeight = originalHeight;
if (this.config.filter.width > 0) targetWidth = this.config.filter.width;
else if (this.config.filter.height > 0) targetWidth = originalWidth * (this.config.filter.height / originalHeight);
if (this.config.filter.height > 0) targetHeight = this.config.filter.height;
else if (this.config.filter.width > 0) targetHeight = originalHeight * (this.config.filter.width / originalWidth);
if (!this.inCanvas || (this.inCanvas.width !== originalWidth) || (this.inCanvas.height !== originalHeight)) {
this.inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
if (this.inCanvas.width !== targetWidth) this.inCanvas.width = targetWidth;
if (this.inCanvas.height !== targetHeight) this.inCanvas.height = targetHeight;
}
const ctx = this.inCanvas.getContext('2d');
if (input instanceof ImageData) ctx.putImageData(input, 0, 0);
else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, this.inCanvas.width, this.inCanvas.height);
if (this.config.filter.enabled) {
if (!this.outCanvas || (this.inCanvas.width !== this.outCanvas.width) || (this.inCanvas.height !== this.outCanvas.height)) {
this.outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(this.inCanvas.width, this.inCanvas.height) : document.createElement('canvas');
if (this.outCanvas.width !== this.inCanvas.width) this.outCanvas.width = this.inCanvas.width;
if (this.outCanvas.height !== this.inCanvas.height) this.outCanvas.height = this.inCanvas.height;
}
if (!this.fx) this.fx = (tf.ENV.flags.IS_BROWSER && (typeof document !== 'undefined')) ? new fxImage.Canvas({ canvas: this.outCanvas }) : null;
this.fx.reset();
this.fx.addFilter('brightness', this.config.filter.brightness); // must have at least one filter enabled
if (this.config.filter.contrast !== 0) this.fx.addFilter('contrast', this.config.filter.contrast);
if (this.config.filter.sharpness !== 0) this.fx.addFilter('sharpen', this.config.filter.sharpness);
if (this.config.filter.blur !== 0) this.fx.addFilter('blur', this.config.filter.blur);
if (this.config.filter.saturation !== 0) this.fx.addFilter('saturation', this.config.filter.saturation);
if (this.config.filter.hue !== 0) this.fx.addFilter('hue', this.config.filter.hue);
if (this.config.filter.negative) this.fx.addFilter('negative');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.vintage) this.fx.addFilter('brownie');
if (this.config.filter.sepia) this.fx.addFilter('sepia');
if (this.config.filter.kodachrome) this.fx.addFilter('kodachrome');
if (this.config.filter.technicolor) this.fx.addFilter('technicolor');
if (this.config.filter.polaroid) this.fx.addFilter('polaroid');
if (this.config.filter.pixelate !== 0) this.fx.addFilter('pixelate', this.config.filter.pixelate);
this.fx.apply(this.inCanvas);
}
if (!this.outCanvas) this.outCanvas = this.inCanvas;
let pixels;
if ((this.config.backend === 'webgl') || (canvas instanceof ImageData)) {
if ((this.config.backend === 'webgl') || (this.outCanvas instanceof ImageData)) {
// tf kernel-optimized method to get imagedata, also if input is imagedata, just use it
pixels = tf.browser.fromPixels(canvas);
pixels = tf.browser.fromPixels(this.outCanvas);
} else {
// cpu and wasm kernel does not implement efficient fromPixels method nor we can use canvas as-is, so we do a silly one more canvas
const tempCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext('2d');
tempCtx.drawImage(canvas, 0, 0);
tempCtx.drawImage(this.outCanvas, 0, 0);
const data = tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf.browser.fromPixels(data);
}
@ -218,7 +227,7 @@ class Human {
pixels.dispose();
casted.dispose();
}
return { tensor, canvas: this.config.filter.return ? filtered : null };
return { tensor, canvas: this.config.filter.return ? this.outCanvas : null };
}
async detect(input, userConfig = {}) {
@ -239,6 +248,11 @@ class Human {
// eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve) => {
let poseRes;
let handRes;
let ssrRes;
let emotionRes;
const timeStart = now();
// configure backend
@ -270,20 +284,30 @@ class Human {
const imageTensor = image.tensor;
// run posenet
this.state = 'run:body';
timeStamp = now();
this.analyze('Start PoseNet');
const poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze('End PoseNet:');
perf.body = Math.trunc(now() - timeStamp);
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = 'run:body';
timeStamp = now();
this.analyze('Start PoseNet');
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze('End PoseNet:');
perf.body = Math.trunc(now() - timeStamp);
}
// run handpose
this.state = 'run:hand';
timeStamp = now();
this.analyze('Start HandPose:');
const handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze('End HandPose:');
perf.hand = Math.trunc(now() - timeStamp);
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = 'run:hand';
timeStamp = now();
this.analyze('Start HandPose:');
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze('End HandPose:');
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async) [poseRes, handRes] = await Promise.all([poseRes, handRes]);
// run facemesh, includes blazeface and iris
const faceRes = [];
@ -302,12 +326,12 @@ class Human {
// run ssr-net age & gender, inherits face from blazeface
this.state = 'run:agegender';
timeStamp = now();
const ssrData = (this.config.face.age.enabled || this.config.face.gender.enabled) ? await ssrnet.predict(face.image, this.config) : {};
ssrRes = (this.config.face.age.enabled || this.config.face.gender.enabled) ? await ssrnet.predict(face.image, this.config) : {};
perf.agegender = Math.trunc(now() - timeStamp);
// run emotion, inherits face from blazeface
this.state = 'run:emotion';
timeStamp = now();
const emotionData = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
perf.emotion = Math.trunc(now() - timeStamp);
// dont need face anymore
@ -322,10 +346,10 @@ class Human {
box: face.box,
mesh: face.mesh,
annotations: face.annotations,
age: ssrData.age,
gender: ssrData.gender,
agConfidence: ssrData.confidence,
emotion: emotionData,
age: ssrRes.age,
gender: ssrRes.gender,
agConfidence: ssrRes.confidence,
emotion: emotionRes,
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
});
this.analyze('End FaceMesh:');

6
src/ssrnet/ssrnet.js
View File

@ -30,7 +30,7 @@ async function predict(image, config) {
let genderT;
const obj = {};
if (!config.profile) {
if (!config.profile || config.async) {
if (config.face.age.enabled) promises.push(ageT = models.age.predict(enhance));
if (config.face.gender.enabled) promises.push(genderT = models.gender.predict(enhance));
await Promise.all(promises);
@ -46,12 +46,12 @@ async function predict(image, config) {
}
if (ageT) {
const data = await ageT.data();
const data = ageT.dataSync();
obj.age = Math.trunc(10 * data[0]) / 10;
tf.dispose(ageT);
}
if (genderT) {
const data = await genderT.data();
const data = genderT.dataSync();
const confidence = Math.trunc(Math.abs(1.9 * 100 * (data[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = data[0] <= 0.5 ? 'female' : 'male';