mirror of https://github.com/vladmandic/human
fixed memory leaks and added scoped runs
Vladimir Mandic
parent
12de5a71b5
commit
5754e5e36e
|
|
@ -4,6 +4,9 @@
|
|||
export default {
|
||||
backend: 'webgl', // select tfjs backend to use
|
||||
console: true, // enable debugging output to console
|
||||
scoped: false, // enable scoped runs
|
||||
// some models *may* have memory leaks, this wrapps everything in a local scope at a cost of performance
|
||||
// typically not needed
|
||||
face: {
|
||||
enabled: true, // controls if specified modul is enabled
|
||||
// face.enabled is required for all face models: detector, mesh, iris, age, gender, emotion
|
||||
|
|
|
|||
119
demo/browser.js
119
demo/browser.js
|
|
@ -3,6 +3,7 @@
|
|||
import human from '../dist/human.esm.js';
|
||||
import draw from './draw.js';
|
||||
|
||||
// ui options
|
||||
const ui = {
|
||||
baseColor: 'rgba(255, 200, 255, 0.3)',
|
||||
baseLabel: 'rgba(255, 200, 255, 0.9)',
|
||||
|
|
@ -20,11 +21,11 @@ const ui = {
|
|||
drawPolygons: true,
|
||||
fillPolygons: true,
|
||||
useDepth: true,
|
||||
console: true,
|
||||
};
|
||||
|
||||
// configuration overrides
|
||||
const config = {
|
||||
backend: 'webgl',
|
||||
console: true,
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
|
|
@ -37,11 +38,14 @@ const config = {
|
|||
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
|
||||
hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
|
||||
};
|
||||
|
||||
// global variables
|
||||
let settings;
|
||||
let worker;
|
||||
let timeStamp;
|
||||
const fps = [];
|
||||
|
||||
// helper function: translates json to human readable string
|
||||
function str(...msg) {
|
||||
if (!Array.isArray(msg)) return msg;
|
||||
let line = '';
|
||||
|
|
@ -52,11 +56,13 @@ function str(...msg) {
|
|||
return line;
|
||||
}
|
||||
|
||||
// helper function: wrapper around console output
|
||||
const log = (...msg) => {
|
||||
// eslint-disable-next-line no-console
|
||||
if (config.console) console.log(...msg);
|
||||
if (ui.console) console.log(...msg);
|
||||
};
|
||||
|
||||
// draws processed results and starts processing of a next frame
|
||||
async function drawResults(input, result, canvas) {
|
||||
// update fps
|
||||
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
|
||||
|
|
@ -84,53 +90,7 @@ async function drawResults(input, result, canvas) {
|
|||
`;
|
||||
}
|
||||
|
||||
// simple wrapper for worker.postmessage that creates worker if one does not exist
|
||||
function webWorker(input, image, canvas) {
|
||||
if (!worker) {
|
||||
// create new webworker and add event handler only once
|
||||
log('Creating worker thread');
|
||||
worker = new Worker(ui.worker, { type: 'module' });
|
||||
// after receiving message from webworker, parse&draw results and send new frame for processing
|
||||
worker.addEventListener('message', (msg) => drawResults(input, msg.data, canvas));
|
||||
}
|
||||
// pass image data as arraybuffer to worker by reference to avoid copy
|
||||
worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]);
|
||||
}
|
||||
|
||||
async function runHumanDetect(input, canvas) {
|
||||
timeStamp = performance.now();
|
||||
// perform detect if live video or not video at all
|
||||
if (input.srcObject) {
|
||||
// if video not ready, just redo
|
||||
const live = (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
|
||||
if (!live) {
|
||||
if (!input.paused) log(`Video not ready: state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
|
||||
setTimeout(() => runHumanDetect(input, canvas), 500);
|
||||
return;
|
||||
}
|
||||
if (ui.useWorker) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
// perform detection in worker
|
||||
webWorker(input, data, canvas);
|
||||
} else {
|
||||
let result = {};
|
||||
try {
|
||||
// perform detection
|
||||
result = await human.detect(input, config);
|
||||
} catch (err) {
|
||||
log('Error during execution:', err.message);
|
||||
}
|
||||
if (result.error) log(result.error);
|
||||
else drawResults(input, result, canvas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
// setup webcam
|
||||
async function setupCamera() {
|
||||
if (ui.busy) return null;
|
||||
ui.busy = true;
|
||||
|
|
@ -173,12 +133,55 @@ async function setupCamera() {
|
|||
});
|
||||
}
|
||||
|
||||
// wrapper for worker.postmessage that creates worker if one does not exist
|
||||
function webWorker(input, image, canvas) {
|
||||
if (!worker) {
|
||||
// create new webworker and add event handler only once
|
||||
log('Creating worker thread');
|
||||
worker = new Worker(ui.worker, { type: 'module' });
|
||||
// after receiving message from webworker, parse&draw results and send new frame for processing
|
||||
worker.addEventListener('message', (msg) => drawResults(input, msg.data, canvas));
|
||||
}
|
||||
// pass image data as arraybuffer to worker by reference to avoid copy
|
||||
worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]);
|
||||
}
|
||||
|
||||
// main processing function when input is webcam, can use direct invocation or web worker
|
||||
async function runHumanDetect(input, canvas) {
|
||||
timeStamp = performance.now();
|
||||
// perform detect if live video or not video at all
|
||||
if (input.srcObject) {
|
||||
// if video not ready, just redo
|
||||
const live = (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
|
||||
if (!live) {
|
||||
if (!input.paused) log(`Video not ready: state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`);
|
||||
setTimeout(() => runHumanDetect(input, canvas), 500);
|
||||
return;
|
||||
}
|
||||
if (ui.useWorker) {
|
||||
// get image data from video as we cannot send html objects to webworker
|
||||
const offscreen = new OffscreenCanvas(canvas.width, canvas.height);
|
||||
const ctx = offscreen.getContext('2d');
|
||||
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
|
||||
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
// perform detection in worker
|
||||
webWorker(input, data, canvas);
|
||||
} else {
|
||||
let result = {};
|
||||
try {
|
||||
// perform detection
|
||||
result = await human.detect(input, config);
|
||||
} catch (err) {
|
||||
log('Error during execution:', err.message);
|
||||
}
|
||||
if (result.error) log(result.error);
|
||||
else drawResults(input, result, canvas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// main processing function when input is image, can use direct invocation or web worker
|
||||
async function processImage(input) {
|
||||
ui.baseColor = 'rgba(200, 255, 255, 0.5)';
|
||||
ui.baseLabel = 'rgba(200, 255, 255, 0.8)';
|
||||
ui.baseFont = 'small-caps 3.5rem "Segoe UI"';
|
||||
ui.baseLineWidth = 16;
|
||||
ui.columns = 3;
|
||||
const cfg = {
|
||||
backend: 'webgl',
|
||||
console: true,
|
||||
|
|
@ -218,6 +221,7 @@ async function processImage(input) {
|
|||
});
|
||||
}
|
||||
|
||||
// just initialize everything and call main function
|
||||
async function detectVideo() {
|
||||
document.getElementById('samples').style.display = 'none';
|
||||
document.getElementById('canvas').style.display = 'block';
|
||||
|
|
@ -236,7 +240,7 @@ async function detectVideo() {
|
|||
runHumanDetect(video, canvas);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
// just initialize everything and call main function
|
||||
async function detectSampleImages() {
|
||||
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${ui.columns}rem`);
|
||||
ui.baseLineHeight = ui.baseLineHeightProto * ui.columns;
|
||||
|
|
@ -246,8 +250,8 @@ async function detectSampleImages() {
|
|||
for (const sample of ui.samples) await processImage(sample);
|
||||
}
|
||||
|
||||
// setup settings panel
|
||||
function setupUI() {
|
||||
// add all variables to ui control panel
|
||||
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
|
||||
const style = document.createElement('style');
|
||||
style.innerHTML = `
|
||||
|
|
@ -314,7 +318,6 @@ function setupUI() {
|
|||
async function main() {
|
||||
log('Human demo starting ...');
|
||||
setupUI();
|
||||
|
||||
const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`;
|
||||
document.getElementById('log').innerText += '\n' + msg;
|
||||
log(msg);
|
||||
|
|
|
|||
|
|
@ -113,8 +113,8 @@ var require_blazeface = __commonJS((exports2) => {
|
|||
const boxIndicesTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
|
||||
const boxIndices = await boxIndicesTensor.array();
|
||||
boxIndicesTensor.dispose();
|
||||
let boundingBoxes = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
boundingBoxes = await Promise.all(boundingBoxes.map(async (boundingBox) => {
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf2.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
|
||||
const vals = await boundingBox.array();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
|
|
@ -122,16 +122,19 @@ var require_blazeface = __commonJS((exports2) => {
|
|||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boundingBox = boundingBoxes[i];
|
||||
const annotatedBox = tf2.tidy(() => {
|
||||
const box = createBox(boundingBox);
|
||||
const boxIndex = boxIndices[i];
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf2.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]).squeeze().reshape([NUM_LANDMARKS, -1]);
|
||||
const probability = tf2.slice(scores, [boxIndex], [1]);
|
||||
return {box, landmarks, probability, anchor};
|
||||
});
|
||||
const box = createBox(boundingBox);
|
||||
const boxIndex = boxIndices[i];
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const sliced = tf2.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]);
|
||||
const squeezed = sliced.squeeze();
|
||||
const landmarks = squeezed.reshape([NUM_LANDMARKS, -1]);
|
||||
const probability = tf2.slice(scores, [boxIndex], [1]);
|
||||
const annotatedBox = {box, landmarks, probability, anchor};
|
||||
annotatedBoxes.push(annotatedBox);
|
||||
sliced.dispose();
|
||||
squeezed.dispose();
|
||||
}
|
||||
detectedOutputs.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
detectedOutputs.dispose();
|
||||
|
|
@ -141,12 +144,11 @@ var require_blazeface = __commonJS((exports2) => {
|
|||
};
|
||||
}
|
||||
async estimateFaces(input) {
|
||||
const image = tf2.tidy(() => {
|
||||
if (!(input instanceof tf2.Tensor)) {
|
||||
input = tf2.browser.fromPixels(input);
|
||||
}
|
||||
return input.toFloat().expandDims(0);
|
||||
});
|
||||
const imageRaw = !(input instanceof tf2.Tensor) ? tf2.browser.fromPixels(input) : input;
|
||||
const imageCast = imageRaw.toFloat();
|
||||
const image = imageCast.expandDims(0);
|
||||
imageRaw.dispose();
|
||||
imageCast.dispose();
|
||||
const {boxes, scaleFactor} = await this.getBoundingBoxes(image);
|
||||
image.dispose();
|
||||
return Promise.all(boxes.map(async (face) => {
|
||||
|
|
@ -172,12 +174,12 @@ var require_blazeface = __commonJS((exports2) => {
|
|||
}));
|
||||
}
|
||||
}
|
||||
async function load(config2) {
|
||||
async function load2(config2) {
|
||||
const blazeface = await tf2.loadGraphModel(config2.detector.modelPath, {fromTFHub: config2.detector.modelPath.includes("tfhub.dev")});
|
||||
const model = new BlazeFaceModel(blazeface, config2);
|
||||
return model;
|
||||
}
|
||||
exports2.load = load;
|
||||
exports2.load = load2;
|
||||
exports2.BlazeFaceModel = BlazeFaceModel;
|
||||
exports2.disposeBox = disposeBox;
|
||||
});
|
||||
|
|
@ -530,21 +532,25 @@ var require_pipeline = __commonJS((exports2) => {
|
|||
this.skipFrames = config2.detector.skipFrames;
|
||||
this.maxFaces = config2.detector.maxFaces;
|
||||
if (this.shouldUpdateRegionsOfInterest()) {
|
||||
const {boxes, scaleFactor} = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
if (boxes.length === 0) {
|
||||
const detector = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
if (detector.boxes.length === 0) {
|
||||
this.regionsOfInterest = [];
|
||||
return null;
|
||||
}
|
||||
const scaledBoxes = boxes.map((prediction) => {
|
||||
const scaledBoxes = detector.boxes.map((prediction) => {
|
||||
const startPoint = prediction.box.startPoint.squeeze();
|
||||
const endPoint = prediction.box.endPoint.squeeze();
|
||||
const predictionBox = {
|
||||
startPoint: prediction.box.startPoint.squeeze().arraySync(),
|
||||
endPoint: prediction.box.endPoint.squeeze().arraySync()
|
||||
startPoint: startPoint.arraySync(),
|
||||
endPoint: endPoint.arraySync()
|
||||
};
|
||||
prediction.box.startPoint.dispose();
|
||||
prediction.box.endPoint.dispose();
|
||||
const scaledBox = bounding.scaleBoxCoordinates(predictionBox, scaleFactor);
|
||||
startPoint.dispose();
|
||||
endPoint.dispose();
|
||||
const scaledBox = bounding.scaleBoxCoordinates(predictionBox, detector.scaleFactor);
|
||||
const enlargedBox = bounding.enlargeBox(scaledBox);
|
||||
const landmarks = prediction.landmarks.arraySync();
|
||||
prediction.box.startPoint.dispose();
|
||||
prediction.box.endPoint.dispose();
|
||||
prediction.landmarks.dispose();
|
||||
prediction.probability.dispose();
|
||||
return {...enlargedBox, landmarks};
|
||||
|
|
@ -601,13 +607,15 @@ var require_pipeline = __commonJS((exports2) => {
|
|||
const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
|
||||
tf2.dispose(rawCoords);
|
||||
const landmarksBox = bounding.enlargeBox(this.calculateLandmarksBoundingBox(transformedCoordsData));
|
||||
const confidence = flag.squeeze();
|
||||
tf2.dispose(flag);
|
||||
if (config2.mesh.enabled) {
|
||||
const transformedCoords = tf2.tensor2d(transformedCoordsData);
|
||||
this.regionsOfInterest[i] = {...landmarksBox, landmarks: transformedCoords.arraySync()};
|
||||
const prediction2 = {
|
||||
coords: transformedCoords,
|
||||
box: landmarksBox,
|
||||
confidence: flag.squeeze(),
|
||||
confidence,
|
||||
image: face
|
||||
};
|
||||
return prediction2;
|
||||
|
|
@ -615,7 +623,7 @@ var require_pipeline = __commonJS((exports2) => {
|
|||
const prediction = {
|
||||
coords: null,
|
||||
box: landmarksBox,
|
||||
confidence: flag.squeeze(),
|
||||
confidence,
|
||||
image: face
|
||||
};
|
||||
return prediction;
|
||||
|
|
@ -668,7 +676,7 @@ var require_pipeline = __commonJS((exports2) => {
|
|||
const ys = landmarks.map((d) => d[1]);
|
||||
const startPoint = [Math.min(...xs), Math.min(...ys)];
|
||||
const endPoint = [Math.max(...xs), Math.max(...ys)];
|
||||
return {startPoint, endPoint};
|
||||
return {startPoint, endPoint, landmarks};
|
||||
}
|
||||
}
|
||||
exports2.Pipeline = Pipeline;
|
||||
|
|
@ -3814,11 +3822,11 @@ var require_facemesh = __commonJS((exports2) => {
|
|||
async estimateFaces(input, config2) {
|
||||
if (config2)
|
||||
this.config = config2;
|
||||
const image = tf2.tidy(() => {
|
||||
if (!(input instanceof tf2.Tensor))
|
||||
input = tf2.browser.fromPixels(input);
|
||||
return input.toFloat().expandDims(0);
|
||||
});
|
||||
const imageRaw = !(input instanceof tf2.Tensor) ? tf2.browser.fromPixels(input) : input;
|
||||
const imageCast = imageRaw.toFloat();
|
||||
const image = imageCast.expandDims(0);
|
||||
imageRaw.dispose();
|
||||
imageCast.dispose();
|
||||
const predictions = await this.pipeline.predict(image, config2);
|
||||
tf2.dispose(image);
|
||||
const results = [];
|
||||
|
|
@ -3844,13 +3852,17 @@ var require_facemesh = __commonJS((exports2) => {
|
|||
image: prediction.image ? tf2.clone(prediction.image) : null
|
||||
});
|
||||
}
|
||||
prediction.confidence.dispose();
|
||||
prediction.image.dispose();
|
||||
if (prediction.confidence)
|
||||
prediction.confidence.dispose();
|
||||
if (prediction.coords)
|
||||
prediction.coords.dispose();
|
||||
if (prediction.image)
|
||||
prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
async function load(config2) {
|
||||
async function load2(config2) {
|
||||
const models2 = await Promise.all([
|
||||
blazeface.load(config2),
|
||||
tf2.loadGraphModel(config2.mesh.modelPath, {fromTFHub: config2.mesh.modelPath.includes("tfhub.dev")}),
|
||||
|
|
@ -3859,7 +3871,7 @@ var require_facemesh = __commonJS((exports2) => {
|
|||
const faceMesh = new MediaPipeFaceMesh(models2[0], models2[1], models2[2], config2);
|
||||
return faceMesh;
|
||||
}
|
||||
exports2.load = load;
|
||||
exports2.load = load2;
|
||||
exports2.MediaPipeFaceMesh = MediaPipeFaceMesh;
|
||||
exports2.uv_coords = uv_coords;
|
||||
exports2.triangulation = triangulation;
|
||||
|
|
@ -3946,7 +3958,7 @@ var require_emotion = __commonJS((exports2) => {
|
|||
});
|
||||
return tensor;
|
||||
}
|
||||
async function load(config2) {
|
||||
async function load2(config2) {
|
||||
if (!models2.emotion)
|
||||
models2.emotion = await tf2.loadGraphModel(config2.face.emotion.modelPath);
|
||||
return models2.emotion;
|
||||
|
|
@ -3988,7 +4000,7 @@ var require_emotion = __commonJS((exports2) => {
|
|||
return obj;
|
||||
}
|
||||
exports2.predict = predict;
|
||||
exports2.load = load;
|
||||
exports2.load = load2;
|
||||
});
|
||||
|
||||
// src/posenet/modelBase.js
|
||||
|
|
@ -4542,10 +4554,10 @@ var require_modelPoseNet = __commonJS((exports2) => {
|
|||
const mobilenet = new modelMobileNet.MobileNet(graphModel, config2.outputStride);
|
||||
return new PoseNet(mobilenet);
|
||||
}
|
||||
async function load(config2) {
|
||||
async function load2(config2) {
|
||||
return loadMobileNet(config2);
|
||||
}
|
||||
exports2.load = load;
|
||||
exports2.load = load2;
|
||||
});
|
||||
|
||||
// src/posenet/posenet.js
|
||||
|
|
@ -4681,19 +4693,7 @@ var require_handdetector = __commonJS((exports2) => {
|
|||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
const boxesWithHandsTensor = await tf2.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
|
||||
const boxesWithHands = await boxesWithHandsTensor.array();
|
||||
const toDispose = [
|
||||
normalizedInput,
|
||||
batchedPrediction,
|
||||
boxesWithHandsTensor,
|
||||
prediction,
|
||||
boxes,
|
||||
rawBoxes,
|
||||
scores
|
||||
];
|
||||
if (boxesWithHands.length === 0) {
|
||||
toDispose.forEach((tensor) => tensor.dispose());
|
||||
return null;
|
||||
}
|
||||
const toDispose = [normalizedInput, batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
|
||||
const detectedHands = tf2.tidy(() => {
|
||||
const detectedBoxes = [];
|
||||
for (const i in boxesWithHands) {
|
||||
|
|
@ -4705,6 +4705,7 @@ var require_handdetector = __commonJS((exports2) => {
|
|||
}
|
||||
return detectedBoxes;
|
||||
});
|
||||
toDispose.forEach((tensor) => tensor.dispose());
|
||||
return detectedHands;
|
||||
}
|
||||
async estimateHandBounds(input, config2) {
|
||||
|
|
@ -5033,7 +5034,7 @@ var require_handpose = __commonJS((exports2) => {
|
|||
}
|
||||
return tf2.util.fetch(url).then((d) => d.json());
|
||||
}
|
||||
async function load(config2) {
|
||||
async function load2(config2) {
|
||||
const [anchors, handDetectorModel, handPoseModel] = await Promise.all([
|
||||
loadAnchors(config2.detector.anchors),
|
||||
tf2.loadGraphModel(config2.detector.modelPath, {fromTFHub: config2.detector.modelPath.includes("tfhub.dev")}),
|
||||
|
|
@ -5044,7 +5045,7 @@ var require_handpose = __commonJS((exports2) => {
|
|||
const handpose2 = new HandPose(pipeline);
|
||||
return handpose2;
|
||||
}
|
||||
exports2.load = load;
|
||||
exports2.load = load2;
|
||||
});
|
||||
|
||||
// config.js
|
||||
|
|
@ -5055,6 +5056,7 @@ var require_config = __commonJS((exports2) => {
|
|||
var config_default = {
|
||||
backend: "webgl",
|
||||
console: true,
|
||||
scoped: false,
|
||||
face: {
|
||||
enabled: true,
|
||||
detector: {
|
||||
|
|
@ -5202,6 +5204,7 @@ const handpose = require_handpose();
|
|||
const defaults = require_config().default;
|
||||
const app = require_package();
|
||||
let config;
|
||||
let state = "idle";
|
||||
const models = {
|
||||
facemesh: null,
|
||||
posenet: null,
|
||||
|
|
@ -5217,9 +5220,21 @@ const now = () => {
|
|||
return parseInt(Number(process.hrtime.bigint()) / 1e3 / 1e3);
|
||||
};
|
||||
const log = (...msg) => {
|
||||
if (config.console)
|
||||
if (msg && config.console)
|
||||
console.log(...msg);
|
||||
};
|
||||
let numTensors = 0;
|
||||
const analyzeMemoryLeaks = false;
|
||||
const analyze = (...msg) => {
|
||||
if (!analyzeMemoryLeaks)
|
||||
return;
|
||||
const current = tf.engine().state.numTensors;
|
||||
const previous = numTensors;
|
||||
numTensors = current;
|
||||
const leaked = current - previous;
|
||||
if (leaked !== 0)
|
||||
log(...msg, leaked);
|
||||
};
|
||||
function mergeDeep(...objects) {
|
||||
const isObject = (obj) => obj && typeof obj === "object";
|
||||
return objects.reduce((prev, obj) => {
|
||||
|
|
@ -5252,8 +5267,26 @@ function sanity(input) {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
async function detect(input, userConfig) {
|
||||
async function load(userConfig) {
|
||||
if (userConfig)
|
||||
config = mergeDeep(defaults, userConfig);
|
||||
if (config.face.enabled && !models.facemesh)
|
||||
models.facemesh = await facemesh.load(config.face);
|
||||
if (config.body.enabled && !models.posenet)
|
||||
models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose)
|
||||
models.handpose = await handpose.load(config.hand);
|
||||
if (config.face.enabled && config.face.age.enabled && !models.age)
|
||||
models.age = await ssrnet.loadAge(config);
|
||||
if (config.face.enabled && config.face.gender.enabled && !models.gender)
|
||||
models.gender = await ssrnet.loadGender(config);
|
||||
if (config.face.enabled && config.face.emotion.enabled && !models.emotion)
|
||||
models.emotion = await emotion.load(config);
|
||||
}
|
||||
async function detect(input, userConfig = {}) {
|
||||
state = "config";
|
||||
config = mergeDeep(defaults, userConfig);
|
||||
state = "check";
|
||||
const error = sanity(input);
|
||||
if (error) {
|
||||
log(error, input);
|
||||
|
|
@ -5264,34 +5297,35 @@ async function detect(input, userConfig) {
|
|||
if (loadedModels === 0)
|
||||
log("Human library starting");
|
||||
if (tf.getBackend() !== config.backend) {
|
||||
state = "backend";
|
||||
log("Human library setting backend:", config.backend);
|
||||
await tf.setBackend(config.backend);
|
||||
await tf.ready();
|
||||
}
|
||||
if (config.face.enabled && !models.facemesh)
|
||||
models.facemesh = await facemesh.load(config.face);
|
||||
if (config.body.enabled && !models.posenet)
|
||||
models.posenet = await posenet.load(config.body);
|
||||
if (config.hand.enabled && !models.handpose)
|
||||
models.handpose = await handpose.load(config.hand);
|
||||
if (config.face.enabled && config.face.age.enabled && !models.age)
|
||||
models.age = await ssrnet.loadAge(config);
|
||||
if (config.face.enabled && config.face.gender.enabled && !models.gender)
|
||||
models.gender = await ssrnet.loadGender(config);
|
||||
if (config.face.enabled && config.face.emotion.enabled && !models.emotion)
|
||||
models.emotion = await emotion.load(config);
|
||||
state = "load";
|
||||
await load();
|
||||
const perf = {};
|
||||
let timeStamp;
|
||||
tf.engine().startScope();
|
||||
if (config.scoped)
|
||||
tf.engine().startScope();
|
||||
analyze("Start Detect:");
|
||||
state = "run:body";
|
||||
timeStamp = now();
|
||||
analyze("Start PoseNet");
|
||||
const poseRes = config.body.enabled ? await models.posenet.estimatePoses(input, config.body) : [];
|
||||
analyze("End PoseNet:");
|
||||
perf.body = Math.trunc(now() - timeStamp);
|
||||
state = "run:hand";
|
||||
timeStamp = now();
|
||||
analyze("Start HandPose:");
|
||||
const handRes = config.hand.enabled ? await models.handpose.estimateHands(input, config.hand) : [];
|
||||
analyze("End HandPose:");
|
||||
perf.hand = Math.trunc(now() - timeStamp);
|
||||
const faceRes = [];
|
||||
if (config.face.enabled) {
|
||||
state = "run:face";
|
||||
timeStamp = now();
|
||||
analyze("Start FaceMesh:");
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
|
|
@ -5299,13 +5333,16 @@ async function detect(input, userConfig) {
|
|||
log("face object is disposed:", face.image);
|
||||
continue;
|
||||
}
|
||||
state = "run:agegender";
|
||||
timeStamp = now();
|
||||
const ssrData = config.face.age.enabled || config.face.gender.enabled ? await ssrnet.predict(face.image, config) : {};
|
||||
perf.agegender = Math.trunc(now() - timeStamp);
|
||||
state = "run:emotion";
|
||||
timeStamp = now();
|
||||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(now() - timeStamp);
|
||||
face.image.dispose();
|
||||
delete face.image;
|
||||
const iris = face.annotations.leftEyeIris && face.annotations.rightEyeIris ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) : 0;
|
||||
faceRes.push({
|
||||
confidence: face.confidence,
|
||||
|
|
@ -5319,8 +5356,12 @@ async function detect(input, userConfig) {
|
|||
iris: iris !== 0 ? Math.trunc(100 * 11.7 / iris) / 100 : 0
|
||||
});
|
||||
}
|
||||
analyze("End FaceMesh:");
|
||||
}
|
||||
tf.engine().endScope();
|
||||
state = "idle";
|
||||
if (config.scoped)
|
||||
tf.engine().endScope();
|
||||
analyze("End Scope:");
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
resolve({face: faceRes, body: poseRes, hand: handRes, performance: perf});
|
||||
});
|
||||
|
|
@ -5335,4 +5376,5 @@ exports.posenet = posenet;
|
|||
exports.handpose = handpose;
|
||||
exports.tf = tf;
|
||||
exports.version = app.version;
|
||||
exports.state = state;
|
||||
//# sourceMappingURL=human.cjs.map
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"config.js": {
|
||||
"bytes": 4536,
|
||||
"bytes": 4774,
|
||||
"imports": []
|
||||
},
|
||||
"package.json": {
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytes": 7042,
|
||||
"bytes": 7407,
|
||||
"imports": []
|
||||
},
|
||||
"src/facemesh/box.js": {
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytes": 2649,
|
||||
"bytes": 2816,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/blazeface.js"
|
||||
|
|
@ -45,7 +45,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytes": 14108,
|
||||
"bytes": 14393,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/box.js"
|
||||
|
|
@ -75,7 +75,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 4253,
|
||||
"bytes": 4296,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/handpose/box.js"
|
||||
|
|
@ -116,7 +116,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytes": 6474,
|
||||
"bytes": 7175,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/facemesh.js"
|
||||
|
|
@ -253,13 +253,13 @@
|
|||
"dist/human.cjs.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 216628
|
||||
"bytes": 219147
|
||||
},
|
||||
"dist/human.cjs": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytesInOutput": 7246
|
||||
"bytesInOutput": 7398
|
||||
},
|
||||
"src/facemesh/keypoints.js": {
|
||||
"bytesInOutput": 2771
|
||||
|
|
@ -271,7 +271,7 @@
|
|||
"bytesInOutput": 3027
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytesInOutput": 13162
|
||||
"bytesInOutput": 13366
|
||||
},
|
||||
"src/facemesh/uvcoords.js": {
|
||||
"bytesInOutput": 20586
|
||||
|
|
@ -280,13 +280,13 @@
|
|||
"bytesInOutput": 23311
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytesInOutput": 2758
|
||||
"bytesInOutput": 2950
|
||||
},
|
||||
"src/ssrnet/ssrnet.js": {
|
||||
"bytesInOutput": 2068
|
||||
},
|
||||
"src/emotion/emotion.js": {
|
||||
"bytesInOutput": 2132
|
||||
"bytesInOutput": 2134
|
||||
},
|
||||
"src/posenet/modelBase.js": {
|
||||
"bytesInOutput": 1120
|
||||
|
|
@ -316,7 +316,7 @@
|
|||
"bytesInOutput": 4383
|
||||
},
|
||||
"src/posenet/modelPoseNet.js": {
|
||||
"bytesInOutput": 1974
|
||||
"bytesInOutput": 1976
|
||||
},
|
||||
"src/posenet/posenet.js": {
|
||||
"bytesInOutput": 917
|
||||
|
|
@ -325,7 +325,7 @@
|
|||
"bytesInOutput": 2813
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 4271
|
||||
"bytesInOutput": 4135
|
||||
},
|
||||
"src/handpose/keypoints.js": {
|
||||
"bytesInOutput": 265
|
||||
|
|
@ -337,19 +337,19 @@
|
|||
"bytesInOutput": 7651
|
||||
},
|
||||
"src/handpose/handpose.js": {
|
||||
"bytesInOutput": 2516
|
||||
"bytesInOutput": 2518
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1853
|
||||
"bytesInOutput": 1872
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 2748
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytesInOutput": 5148
|
||||
"bytesInOutput": 6171
|
||||
}
|
||||
},
|
||||
"bytes": 132178
|
||||
"bytes": 133638
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"config.js": {
|
||||
"bytes": 4536,
|
||||
"bytes": 4774,
|
||||
"imports": []
|
||||
},
|
||||
"package.json": {
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytes": 7042,
|
||||
"bytes": 7407,
|
||||
"imports": []
|
||||
},
|
||||
"src/facemesh/box.js": {
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytes": 2649,
|
||||
"bytes": 2816,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/blazeface.js"
|
||||
|
|
@ -45,7 +45,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytes": 14108,
|
||||
"bytes": 14393,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/box.js"
|
||||
|
|
@ -75,7 +75,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 4253,
|
||||
"bytes": 4296,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/handpose/box.js"
|
||||
|
|
@ -116,7 +116,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytes": 6474,
|
||||
"bytes": 7175,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/facemesh/facemesh.js"
|
||||
|
|
@ -253,13 +253,13 @@
|
|||
"dist/human.esm-nobundle.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 194920
|
||||
"bytes": 197443
|
||||
},
|
||||
"dist/human.esm-nobundle.js": {
|
||||
"imports": [],
|
||||
"inputs": {
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytesInOutput": 3223
|
||||
"bytesInOutput": 3255
|
||||
},
|
||||
"src/facemesh/keypoints.js": {
|
||||
"bytesInOutput": 1950
|
||||
|
|
@ -271,7 +271,7 @@
|
|||
"bytesInOutput": 1176
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytesInOutput": 5541
|
||||
"bytesInOutput": 5602
|
||||
},
|
||||
"src/facemesh/uvcoords.js": {
|
||||
"bytesInOutput": 16790
|
||||
|
|
@ -280,7 +280,7 @@
|
|||
"bytesInOutput": 9995
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytesInOutput": 1320
|
||||
"bytesInOutput": 1391
|
||||
},
|
||||
"src/ssrnet/ssrnet.js": {
|
||||
"bytesInOutput": 1099
|
||||
|
|
@ -325,31 +325,31 @@
|
|||
"bytesInOutput": 1400
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 2074
|
||||
"bytesInOutput": 2040
|
||||
},
|
||||
"src/handpose/keypoints.js": {
|
||||
"bytesInOutput": 160
|
||||
},
|
||||
"src/handpose/util.js": {
|
||||
"bytesInOutput": 977
|
||||
"bytesInOutput": 984
|
||||
},
|
||||
"src/handpose/pipeline.js": {
|
||||
"bytesInOutput": 3230
|
||||
"bytesInOutput": 3232
|
||||
},
|
||||
"src/handpose/handpose.js": {
|
||||
"bytesInOutput": 1326
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1136
|
||||
"bytesInOutput": 1146
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 2275
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytesInOutput": 2904
|
||||
"bytesInOutput": 3410
|
||||
}
|
||||
},
|
||||
"bytes": 68538
|
||||
"bytes": 69193
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"config.js": {
|
||||
"bytes": 4536,
|
||||
"bytes": 4774,
|
||||
"imports": []
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
|
||||
|
|
@ -161,7 +161,7 @@
|
|||
]
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytes": 7042,
|
||||
"bytes": 7407,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -177,7 +177,7 @@
|
|||
]
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytes": 2649,
|
||||
"bytes": 2816,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -204,7 +204,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytes": 14108,
|
||||
"bytes": 14393,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -241,7 +241,7 @@
|
|||
]
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 4253,
|
||||
"bytes": 4296,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -291,7 +291,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytes": 6474,
|
||||
"bytes": 7175,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -464,7 +464,7 @@
|
|||
"dist/human.esm.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 4955971
|
||||
"bytes": 4958494
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"imports": [],
|
||||
|
|
@ -527,7 +527,7 @@
|
|||
"bytesInOutput": 765
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytesInOutput": 3238
|
||||
"bytesInOutput": 3268
|
||||
},
|
||||
"src/facemesh/keypoints.js": {
|
||||
"bytesInOutput": 1951
|
||||
|
|
@ -539,7 +539,7 @@
|
|||
"bytesInOutput": 1195
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytesInOutput": 5520
|
||||
"bytesInOutput": 5577
|
||||
},
|
||||
"src/facemesh/uvcoords.js": {
|
||||
"bytesInOutput": 16791
|
||||
|
|
@ -548,7 +548,7 @@
|
|||
"bytesInOutput": 9996
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytesInOutput": 1306
|
||||
"bytesInOutput": 1376
|
||||
},
|
||||
"src/ssrnet/ssrnet.js": {
|
||||
"bytesInOutput": 1100
|
||||
|
|
@ -593,7 +593,7 @@
|
|||
"bytesInOutput": 1386
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 2084
|
||||
"bytesInOutput": 2050
|
||||
},
|
||||
"src/handpose/keypoints.js": {
|
||||
"bytesInOutput": 161
|
||||
|
|
@ -608,16 +608,16 @@
|
|||
"bytesInOutput": 1312
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1137
|
||||
"bytesInOutput": 1147
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 2276
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytesInOutput": 2963
|
||||
"bytesInOutput": 3495
|
||||
}
|
||||
},
|
||||
"bytes": 1105435
|
||||
"bytes": 1106100
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"config.js": {
|
||||
"bytes": 4536,
|
||||
"bytes": 4774,
|
||||
"imports": []
|
||||
},
|
||||
"node_modules/@tensorflow/tfjs-backend-cpu/dist/tf-backend-cpu.node.js": {
|
||||
|
|
@ -161,7 +161,7 @@
|
|||
]
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytes": 7042,
|
||||
"bytes": 7407,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -177,7 +177,7 @@
|
|||
]
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytes": 2649,
|
||||
"bytes": 2816,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -204,7 +204,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytes": 14108,
|
||||
"bytes": 14393,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -241,7 +241,7 @@
|
|||
]
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 4253,
|
||||
"bytes": 4296,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -291,7 +291,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytes": 6474,
|
||||
"bytes": 7175,
|
||||
"imports": [
|
||||
{
|
||||
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
|
||||
|
|
@ -464,7 +464,7 @@
|
|||
"dist/human.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 4955971
|
||||
"bytes": 4958494
|
||||
},
|
||||
"dist/human.js": {
|
||||
"imports": [],
|
||||
|
|
@ -527,7 +527,7 @@
|
|||
"bytesInOutput": 765
|
||||
},
|
||||
"src/facemesh/blazeface.js": {
|
||||
"bytesInOutput": 3238
|
||||
"bytesInOutput": 3268
|
||||
},
|
||||
"src/facemesh/keypoints.js": {
|
||||
"bytesInOutput": 1951
|
||||
|
|
@ -539,7 +539,7 @@
|
|||
"bytesInOutput": 1195
|
||||
},
|
||||
"src/facemesh/pipeline.js": {
|
||||
"bytesInOutput": 5520
|
||||
"bytesInOutput": 5577
|
||||
},
|
||||
"src/facemesh/uvcoords.js": {
|
||||
"bytesInOutput": 16791
|
||||
|
|
@ -548,7 +548,7 @@
|
|||
"bytesInOutput": 9996
|
||||
},
|
||||
"src/facemesh/facemesh.js": {
|
||||
"bytesInOutput": 1306
|
||||
"bytesInOutput": 1376
|
||||
},
|
||||
"src/ssrnet/ssrnet.js": {
|
||||
"bytesInOutput": 1100
|
||||
|
|
@ -593,7 +593,7 @@
|
|||
"bytesInOutput": 1386
|
||||
},
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 2084
|
||||
"bytesInOutput": 2050
|
||||
},
|
||||
"src/handpose/keypoints.js": {
|
||||
"bytesInOutput": 161
|
||||
|
|
@ -608,16 +608,16 @@
|
|||
"bytesInOutput": 1312
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1137
|
||||
"bytesInOutput": 1147
|
||||
},
|
||||
"package.json": {
|
||||
"bytesInOutput": 2276
|
||||
},
|
||||
"src/index.js": {
|
||||
"bytesInOutput": 2963
|
||||
"bytesInOutput": 3495
|
||||
}
|
||||
},
|
||||
"bytes": 1105444
|
||||
"bytes": 1106109
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ class BlazeFaceModel {
|
|||
this.scoreThreshold = config.detector.scoreThreshold;
|
||||
}
|
||||
|
||||
// toto blazeface leaks two tensors per run
|
||||
async getBoundingBoxes(inputImage) {
|
||||
// sanity check on input
|
||||
if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null;
|
||||
|
|
@ -101,12 +102,11 @@ class BlazeFaceModel {
|
|||
const scoresOut = tf.sigmoid(logits).squeeze();
|
||||
return [prediction, decodedBounds, scoresOut];
|
||||
});
|
||||
|
||||
const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxFaces, this.iouThreshold, this.scoreThreshold);
|
||||
const boxIndices = await boxIndicesTensor.array();
|
||||
boxIndicesTensor.dispose();
|
||||
let boundingBoxes = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
boundingBoxes = await Promise.all(boundingBoxes.map(async (boundingBox) => {
|
||||
const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
|
||||
const boundingBoxes = await Promise.all(boundingBoxesMap.map(async (boundingBox) => {
|
||||
const vals = await boundingBox.array();
|
||||
boundingBox.dispose();
|
||||
return vals;
|
||||
|
|
@ -114,19 +114,26 @@ class BlazeFaceModel {
|
|||
const annotatedBoxes = [];
|
||||
for (let i = 0; i < boundingBoxes.length; i++) {
|
||||
const boundingBox = boundingBoxes[i];
|
||||
const annotatedBox = tf.tidy(() => {
|
||||
const box = createBox(boundingBox);
|
||||
const boxIndex = boxIndices[i];
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const landmarks = tf
|
||||
.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1])
|
||||
.squeeze()
|
||||
.reshape([NUM_LANDMARKS, -1]);
|
||||
const probability = tf.slice(scores, [boxIndex], [1]);
|
||||
return { box, landmarks, probability, anchor };
|
||||
});
|
||||
const box = createBox(boundingBox);
|
||||
const boxIndex = boxIndices[i];
|
||||
const anchor = this.anchorsData[boxIndex];
|
||||
const sliced = tf.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1]);
|
||||
const squeezed = sliced.squeeze();
|
||||
const landmarks = squeezed.reshape([NUM_LANDMARKS, -1]);
|
||||
/*
|
||||
const landmarks = tf
|
||||
.slice(detectedOutputs, [boxIndex, NUM_LANDMARKS - 1], [1, -1])
|
||||
.squeeze()
|
||||
.reshape([NUM_LANDMARKS, -1]);
|
||||
*/
|
||||
const probability = tf.slice(scores, [boxIndex], [1]);
|
||||
const annotatedBox = { box, landmarks, probability, anchor };
|
||||
annotatedBoxes.push(annotatedBox);
|
||||
sliced.dispose();
|
||||
squeezed.dispose();
|
||||
// landmarks.dispose();
|
||||
}
|
||||
detectedOutputs.dispose();
|
||||
boxes.dispose();
|
||||
scores.dispose();
|
||||
detectedOutputs.dispose();
|
||||
|
|
@ -137,12 +144,11 @@ class BlazeFaceModel {
|
|||
}
|
||||
|
||||
async estimateFaces(input) {
|
||||
const image = tf.tidy(() => {
|
||||
if (!(input instanceof tf.Tensor)) {
|
||||
input = tf.browser.fromPixels(input);
|
||||
}
|
||||
return input.toFloat().expandDims(0);
|
||||
});
|
||||
const imageRaw = !(input instanceof tf.Tensor) ? tf.browser.fromPixels(input) : input;
|
||||
const imageCast = imageRaw.toFloat();
|
||||
const image = imageCast.expandDims(0);
|
||||
imageRaw.dispose();
|
||||
imageCast.dispose();
|
||||
const { boxes, scaleFactor } = await this.getBoundingBoxes(image);
|
||||
image.dispose();
|
||||
return Promise.all(boxes.map(async (face) => {
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ class MediaPipeFaceMesh {
|
|||
|
||||
async estimateFaces(input, config) {
|
||||
if (config) this.config = config;
|
||||
const image = tf.tidy(() => {
|
||||
if (!(input instanceof tf.Tensor)) input = tf.browser.fromPixels(input);
|
||||
return input.toFloat().expandDims(0);
|
||||
});
|
||||
const imageRaw = !(input instanceof tf.Tensor) ? tf.browser.fromPixels(input) : input;
|
||||
const imageCast = imageRaw.toFloat();
|
||||
const image = imageCast.expandDims(0);
|
||||
imageRaw.dispose();
|
||||
imageCast.dispose();
|
||||
const predictions = await this.pipeline.predict(image, config);
|
||||
tf.dispose(image);
|
||||
const results = [];
|
||||
|
|
@ -42,8 +43,9 @@ class MediaPipeFaceMesh {
|
|||
image: prediction.image ? tf.clone(prediction.image) : null,
|
||||
});
|
||||
}
|
||||
prediction.confidence.dispose();
|
||||
prediction.image.dispose();
|
||||
if (prediction.confidence) prediction.confidence.dispose();
|
||||
if (prediction.coords) prediction.coords.dispose();
|
||||
if (prediction.image) prediction.image.dispose();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -132,21 +132,26 @@ class Pipeline {
|
|||
this.skipFrames = config.detector.skipFrames;
|
||||
this.maxFaces = config.detector.maxFaces;
|
||||
if (this.shouldUpdateRegionsOfInterest()) {
|
||||
const { boxes, scaleFactor } = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
if (boxes.length === 0) {
|
||||
// const { boxes, scaleFactor } = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
const detector = await this.boundingBoxDetector.getBoundingBoxes(input);
|
||||
if (detector.boxes.length === 0) {
|
||||
this.regionsOfInterest = [];
|
||||
return null;
|
||||
}
|
||||
const scaledBoxes = boxes.map((prediction) => {
|
||||
const scaledBoxes = detector.boxes.map((prediction) => {
|
||||
const startPoint = prediction.box.startPoint.squeeze();
|
||||
const endPoint = prediction.box.endPoint.squeeze();
|
||||
const predictionBox = {
|
||||
startPoint: prediction.box.startPoint.squeeze().arraySync(),
|
||||
endPoint: prediction.box.endPoint.squeeze().arraySync(),
|
||||
startPoint: startPoint.arraySync(),
|
||||
endPoint: endPoint.arraySync(),
|
||||
};
|
||||
prediction.box.startPoint.dispose();
|
||||
prediction.box.endPoint.dispose();
|
||||
const scaledBox = bounding.scaleBoxCoordinates(predictionBox, scaleFactor);
|
||||
startPoint.dispose();
|
||||
endPoint.dispose();
|
||||
const scaledBox = bounding.scaleBoxCoordinates(predictionBox, detector.scaleFactor);
|
||||
const enlargedBox = bounding.enlargeBox(scaledBox);
|
||||
const landmarks = prediction.landmarks.arraySync();
|
||||
prediction.box.startPoint.dispose();
|
||||
prediction.box.endPoint.dispose();
|
||||
prediction.landmarks.dispose();
|
||||
prediction.probability.dispose();
|
||||
return { ...enlargedBox, landmarks };
|
||||
|
|
@ -206,13 +211,15 @@ class Pipeline {
|
|||
const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
|
||||
tf.dispose(rawCoords);
|
||||
const landmarksBox = bounding.enlargeBox(this.calculateLandmarksBoundingBox(transformedCoordsData));
|
||||
const confidence = flag.squeeze();
|
||||
tf.dispose(flag);
|
||||
if (config.mesh.enabled) {
|
||||
const transformedCoords = tf.tensor2d(transformedCoordsData);
|
||||
this.regionsOfInterest[i] = { ...landmarksBox, landmarks: transformedCoords.arraySync() };
|
||||
const prediction = {
|
||||
coords: transformedCoords,
|
||||
box: landmarksBox,
|
||||
confidence: flag.squeeze(),
|
||||
confidence,
|
||||
image: face,
|
||||
};
|
||||
return prediction;
|
||||
|
|
@ -220,7 +227,7 @@ class Pipeline {
|
|||
const prediction = {
|
||||
coords: null,
|
||||
box: landmarksBox,
|
||||
confidence: flag.squeeze(),
|
||||
confidence,
|
||||
image: face,
|
||||
};
|
||||
return prediction;
|
||||
|
|
@ -278,7 +285,7 @@ class Pipeline {
|
|||
const ys = landmarks.map((d) => d[1]);
|
||||
const startPoint = [Math.min(...xs), Math.min(...ys)];
|
||||
const endPoint = [Math.max(...xs), Math.max(...ys)];
|
||||
return { startPoint, endPoint };
|
||||
return { startPoint, endPoint, landmarks };
|
||||
}
|
||||
}
|
||||
exports.Pipeline = Pipeline;
|
||||
|
|
|
|||
|
|
@ -42,14 +42,11 @@ class HandDetector {
|
|||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
const boxesWithHandsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold);
|
||||
const boxesWithHands = await boxesWithHandsTensor.array();
|
||||
const toDispose = [
|
||||
normalizedInput, batchedPrediction, boxesWithHandsTensor, prediction,
|
||||
boxes, rawBoxes, scores,
|
||||
];
|
||||
if (boxesWithHands.length === 0) {
|
||||
toDispose.forEach((tensor) => tensor.dispose());
|
||||
return null;
|
||||
}
|
||||
const toDispose = [normalizedInput, batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores];
|
||||
// if (boxesWithHands.length === 0) {
|
||||
// toDispose.forEach((tensor) => tensor.dispose());
|
||||
// return null;
|
||||
// }
|
||||
const detectedHands = tf.tidy(() => {
|
||||
const detectedBoxes = [];
|
||||
for (const i in boxesWithHands) {
|
||||
|
|
@ -61,6 +58,7 @@ class HandDetector {
|
|||
}
|
||||
return detectedBoxes;
|
||||
});
|
||||
toDispose.forEach((tensor) => tensor.dispose());
|
||||
return detectedHands;
|
||||
}
|
||||
|
||||
|
|
|
|||
43
src/index.js
43
src/index.js
|
|
@ -20,17 +20,32 @@ const models = {
|
|||
gender: null,
|
||||
emotion: null,
|
||||
};
|
||||
|
||||
// helper function: gets elapsed time on both browser and nodejs
|
||||
const now = () => {
|
||||
if (typeof performance !== 'undefined') return performance.now();
|
||||
return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000);
|
||||
};
|
||||
|
||||
// helper function: wrapper around console output
|
||||
const log = (...msg) => {
|
||||
// eslint-disable-next-line no-console
|
||||
if (config.console) console.log(...msg);
|
||||
if (msg && config.console) console.log(...msg);
|
||||
};
|
||||
|
||||
// helper function that performs deep merge of multiple objects so it allows full inheriance with overrides
|
||||
// helper function: measure tensor leak
|
||||
let numTensors = 0;
|
||||
const analyzeMemoryLeaks = false;
|
||||
const analyze = (...msg) => {
|
||||
if (!analyzeMemoryLeaks) return;
|
||||
const current = tf.engine().state.numTensors;
|
||||
const previous = numTensors;
|
||||
numTensors = current;
|
||||
const leaked = current - previous;
|
||||
if (leaked !== 0) log(...msg, leaked);
|
||||
};
|
||||
|
||||
// helper function: perform deep merge of multiple objects so it allows full inheriance with overrides
|
||||
function mergeDeep(...objects) {
|
||||
const isObject = (obj) => obj && typeof obj === 'object';
|
||||
return objects.reduce((prev, obj) => {
|
||||
|
|
@ -97,12 +112,6 @@ async function detect(input, userConfig = {}) {
|
|||
await tf.setBackend(config.backend);
|
||||
await tf.ready();
|
||||
}
|
||||
// explictly enable depthwiseconv since it's diasabled by default due to issues with large shaders
|
||||
// let savedWebglPackDepthwiseConvFlag;
|
||||
// if (tf.getBackend() === 'webgl') {
|
||||
// savedWebglPackDepthwiseConvFlag = tf.env().get('WEBGL_PACK_DEPTHWISECONV');
|
||||
// tf.env().set('WEBGL_PACK_DEPTHWISECONV', true);
|
||||
// }
|
||||
|
||||
// load models if enabled
|
||||
state = 'load';
|
||||
|
|
@ -111,18 +120,24 @@ async function detect(input, userConfig = {}) {
|
|||
const perf = {};
|
||||
let timeStamp;
|
||||
|
||||
tf.engine().startScope();
|
||||
if (config.scoped) tf.engine().startScope();
|
||||
|
||||
analyze('Start Detect:');
|
||||
|
||||
// run posenet
|
||||
state = 'run:body';
|
||||
timeStamp = now();
|
||||
analyze('Start PoseNet');
|
||||
const poseRes = config.body.enabled ? await models.posenet.estimatePoses(input, config.body) : [];
|
||||
analyze('End PoseNet:');
|
||||
perf.body = Math.trunc(now() - timeStamp);
|
||||
|
||||
// run handpose
|
||||
state = 'run:hand';
|
||||
timeStamp = now();
|
||||
analyze('Start HandPose:');
|
||||
const handRes = config.hand.enabled ? await models.handpose.estimateHands(input, config.hand) : [];
|
||||
analyze('End HandPose:');
|
||||
perf.hand = Math.trunc(now() - timeStamp);
|
||||
|
||||
// run facemesh, includes blazeface and iris
|
||||
|
|
@ -130,6 +145,7 @@ async function detect(input, userConfig = {}) {
|
|||
if (config.face.enabled) {
|
||||
state = 'run:face';
|
||||
timeStamp = now();
|
||||
analyze('Start FaceMesh:');
|
||||
const faces = await models.facemesh.estimateFaces(input, config.face);
|
||||
perf.face = Math.trunc(now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
|
|
@ -149,6 +165,7 @@ async function detect(input, userConfig = {}) {
|
|||
const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
|
||||
perf.emotion = Math.trunc(now() - timeStamp);
|
||||
face.image.dispose();
|
||||
delete face.image;
|
||||
// calculate iris distance
|
||||
// iris: array[ bottom, left, top, right, center ]
|
||||
const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
|
||||
|
|
@ -166,13 +183,13 @@ async function detect(input, userConfig = {}) {
|
|||
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
|
||||
});
|
||||
}
|
||||
state = 'idle';
|
||||
analyze('End FaceMesh:');
|
||||
}
|
||||
|
||||
// set depthwiseconv to original value
|
||||
// tf.env().set('WEBGL_PACK_DEPTHWISECONV', savedWebglPackDepthwiseConvFlag);
|
||||
state = 'idle';
|
||||
|
||||
tf.engine().endScope();
|
||||
if (config.scoped) tf.engine().endScope();
|
||||
analyze('End Scope:');
|
||||
|
||||
// combine and return results
|
||||
perf.total = Object.values(perf).reduce((a, b) => a + b);
|
||||
|
|
|
|||
Loading…
Reference in New Issue