ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add experimental mb3-centernet object detection

pull/134/head
Vladimir Mandic 2021-05-19 08:27:28 -04:00
parent 271b821ab7
commit fa3ab21215
21 changed files with 227798 additions and 5288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
demo/index.js
View File

@ -9,10 +9,11 @@ import webRTC from './helpers/webrtc.js';
let human; let human;
const userConfig = { const userConfig = {
warmup: 'none', warmup: 'full',
/* /*
backend: 'webgl', backend: 'webgl',
async: true, async: false,
cacheSensitivity: 0,
filter: { filter: {
enabled: false, enabled: false,
flip: false, flip: false,
@ -26,9 +27,9 @@ const userConfig = {
}, },
hand: { enabled: false }, hand: { enabled: false },
gesture: { enabled: false }, gesture: { enabled: false },
body: { enabled: true, modelPath: 'posenet.json' }, body: { enabled: false, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' }, // body: { enabled: true, modelPath: 'blazepose.json' },
// object: { enabled: true }, object: { enabled: false },
*/ */
}; };

19308
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

75933
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

75939
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

262
dist/human.node-gpu.js vendored
View File

@ -127,7 +127,7 @@ var config = {
debug: true, debug: true,
async: true, async: true,
warmup: "full", warmup: "full",
cacheSensitivity: 4e-3, cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -192,7 +192,7 @@ var config = {
hand: { hand: {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 32,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -206,7 +206,7 @@ var config = {
}, },
object: { object: {
enabled: false, enabled: false,
modelPath: "nanodet.json", modelPath: "mb3-centernet.json",
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
@ -236,7 +236,7 @@ function info() {
} }
// src/human.ts // src/human.ts
var tf16 = __toModule(require_tfjs_esm()); var tf17 = __toModule(require_tfjs_esm());
// src/tfjs/backend.ts // src/tfjs/backend.ts
var tf = __toModule(require_tfjs_esm()); var tf = __toModule(require_tfjs_esm());
@ -338,16 +338,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image12, cropSize) { function cutBoxFromImageAndResize(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image12, boxes, [0], cropSize); return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -481,11 +481,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
return tf3.concat2d([startNormalized, endNormalized], concatAxis); return tf3.concat2d([startNormalized, endNormalized], concatAxis);
} }
var BlazeFaceModel = class { var BlazeFaceModel = class {
constructor(model6, config3) { constructor(model7, config3) {
this.model = model6; this.model = model7;
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]); this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
this.anchors = tf3.tensor2d(this.anchorsData); this.anchors = tf3.tensor2d(this.anchorsData);
this.inputSize = model6.inputs[0].shape[2]; this.inputSize = model7.inputs[0].shape[2];
this.config = config3; this.config = config3;
} }
async getBoundingBoxes(inputImage) { async getBoundingBoxes(inputImage) {
@ -534,12 +534,12 @@ var BlazeFaceModel = class {
} }
}; };
async function load(config3) { async function load(config3) {
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")}); const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
const blazeFace = new BlazeFaceModel(model6, config3); const blazeFace = new BlazeFaceModel(model7, config3);
if (!model6 || !model6.modelUrl) if (!model7 || !model7.modelUrl)
log("load model failed:", config3.face.detector.modelPath); log("load model failed:", config3.face.detector.modelPath);
else if (config3.debug) else if (config3.debug)
log("load model:", model6.modelUrl); log("load model:", model7.modelUrl);
return blazeFace; return blazeFace;
} }
@ -4174,7 +4174,7 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image12, config3, idx, count2) { async function predict2(image13, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
@ -4183,7 +4183,7 @@ async function predict2(image12, config3, idx, count2) {
} }
skipped = 0; skipped = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4266,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image12 = tf7.tidy(() => { const image13 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4275,9 +4275,9 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image12; return image13;
} }
async function predict3(image12, config3, idx, count2) { async function predict3(image13, config3, idx, count2) {
var _a, _b; var _a, _b;
if (!model2) if (!model2)
return null; return null;
@ -4287,7 +4287,7 @@ async function predict3(image12, config3, idx, count2) {
} }
skipped2 = 0; skipped2 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image12); const enhanced = enhance(image13);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4844,16 +4844,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image12, cropSize) { function cutBoxFromImageAndResize2(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image12, boxes, [0], cropSize); return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16664,9 +16664,9 @@ var anchors = [
// src/handpose/handdetector.ts // src/handpose/handdetector.ts
var HandDetector = class { var HandDetector = class {
constructor(model6) { constructor(model7) {
var _a; var _a;
this.model = model6; this.model = model7;
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]); this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
this.anchorsTensor = tf10.tensor2d(this.anchors); this.anchorsTensor = tf10.tensor2d(this.anchors);
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2]; this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
@ -16720,9 +16720,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image12, config3); const predictions = await this.getBoxes(image13, config3);
image12.dispose(); image13.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16867,11 +16867,11 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image12, config3) { async estimateHands(image13, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image12, config3); boxes = await this.handDetector.estimateHandBounds(image13, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.skipFrame) if (config3.skipFrame)
@ -16890,8 +16890,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17101,13 +17101,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image12, config3) { async function predict6(image13, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image12.shape[2], height: image12.shape[1]}; const imgSize = { width: image13.shape[2], height: image13.shape[1] };
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17134,7 +17134,7 @@ async function predict6(image12, config3) {
return [{ score, keypoints }]; return [{ score, keypoints }];
} }
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var nanodet_exports = {}; var nanodet_exports = {};
__export(nanodet_exports, { __export(nanodet_exports, {
load: () => load8, load: () => load8,
@ -17142,7 +17142,7 @@ __export(nanodet_exports, {
}); });
var tf14 = __toModule(require_tfjs_esm()); var tf14 = __toModule(require_tfjs_esm());
// src/nanodet/labels.ts // src/object/labels.ts
var labels = [ var labels = [
{ class: 1, label: "person" }, { class: 1, label: "person" },
{ class: 2, label: "bicycle" }, { class: 2, label: "bicycle" },
@ -17226,7 +17226,7 @@ var labels = [
{ class: 80, label: "toothbrush" } { class: 80, label: "toothbrush" }
]; ];
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var model5; var model5;
var last3 = []; var last3 = [];
var skipped3 = Number.MAX_SAFE_INTEGER; var skipped3 = Number.MAX_SAFE_INTEGER;
@ -17299,7 +17299,7 @@ async function process2(res, inputSize, outputShape, config3) {
}); });
} }
res.forEach((t) => tf14.dispose(t)); res.forEach((t) => tf14.dispose(t));
const nmsBoxes = results.map((a) => a.boxRaw); const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
const nmsScores = results.map((a) => a.score); const nmsScores = results.map((a) => a.score);
let nmsIdx = []; let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) { if (nmsBoxes && nmsBoxes.length > 0) {
@ -17310,7 +17310,7 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image12, config3) { async function predict7(image13, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
@ -17319,8 +17319,8 @@ async function predict7(image12, config3) {
} }
skipped3 = 0; skipped3 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image12.shape[2], image12.shape[1]]; const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17335,6 +17335,90 @@ async function predict7(image12, config3) {
}); });
} }
// src/object/centernet.ts
var centernet_exports = {};
__export(centernet_exports, {
load: () => load9,
predict: () => predict8
});
var tf15 = __toModule(require_tfjs_esm());
var model6;
var last4 = [];
var skipped4 = Number.MAX_SAFE_INTEGER;
async function load9(config3) {
if (!model6) {
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
const inputs = Object.values(model6.modelSignature["inputs"]);
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
if (!model6.inputSize)
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
if (!model6 || !model6.modelUrl)
log("load model failed:", config3.object.modelPath);
else if (config3.debug)
log("load model:", model6.modelUrl);
} else if (config3.debug)
log("cached model:", model6.modelUrl);
return model6;
}
async function process3(res, inputSize, outputShape, config3) {
const results = [];
const detections = res.arraySync();
const squeezeT = tf15.squeeze(res);
res.dispose();
const arr = tf15.split(squeezeT, 6, 1);
squeezeT.dispose();
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose());
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose();
scoresT.dispose();
classesT.dispose();
const nms = nmsT.dataSync();
nmsT.dispose();
for (const id of nms) {
const score = detections[0][id][4];
const classVal = detections[0][id][5];
const label = labels[classVal].label;
const boxRaw = [
detections[0][id][0] / inputSize,
detections[0][id][1] / inputSize,
detections[0][id][2] / inputSize,
detections[0][id][3] / inputSize
];
const box4 = [
Math.trunc(boxRaw[0] * outputShape[0]),
Math.trunc(boxRaw[1] * outputShape[1]),
Math.trunc(boxRaw[2] * outputShape[0]),
Math.trunc(boxRaw[3] * outputShape[1])
];
results.push({ score, class: classVal, label, box: box4, boxRaw });
}
return results;
}
async function predict8(image13, config3) {
if (!model6)
return null;
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
skipped4++;
return last4;
}
skipped4 = 0;
return new Promise(async (resolve) => {
const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
let objectT;
if (config3.object.enabled)
objectT = model6.execute(resize, "tower_0/detections");
resize.dispose();
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
last4 = obj;
resolve(obj);
});
}
// src/gesture/gesture.ts // src/gesture/gesture.ts
var body = (res) => { var body = (res) => {
if (!res) if (!res)
@ -17444,7 +17528,7 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var tf15 = __toModule(require_tfjs_esm()); var tf16 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
function GLProgram(gl, vertexSource, fragmentSource) { function GLProgram(gl, vertexSource, fragmentSource) {
@ -17596,8 +17680,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image12) { this.apply = function(image13) {
_resize(image12.width, image12.height); _resize(image13.width, image13.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17606,7 +17690,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -18157,16 +18241,16 @@ var maxSize = 2048;
var inCanvas; var inCanvas;
var outCanvas; var outCanvas;
var fx; var fx;
function process3(input, config3) { function process4(input, config3) {
let tensor; let tensor;
if (!input) if (!input)
throw new Error("Human: Input is missing"); throw new Error("Human: Input is missing");
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) { if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
throw new Error("Human: Input type is not recognized"); throw new Error("Human: Input type is not recognized");
} }
if (input instanceof tf15.Tensor) { if (input instanceof tf16.Tensor) {
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
tensor = tf15.clone(input); tensor = tf16.clone(input);
else else
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`); throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
} else { } else {
@ -18219,7 +18303,7 @@ function process3(input, config3) {
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width; outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height)) if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height; outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null; fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
} }
if (!fx) if (!fx)
return { tensor: null, canvas: inCanvas }; return { tensor: null, canvas: inCanvas };
@ -18260,16 +18344,16 @@ function process3(input, config3) {
let pixels; let pixels;
if (outCanvas.data) { if (outCanvas.data) {
const shape = [outCanvas.height, outCanvas.width, 3]; const shape = [outCanvas.height, outCanvas.width, 3];
pixels = tf15.tensor3d(outCanvas.data, shape, "int32"); pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
} else if (outCanvas instanceof ImageData) { } else if (outCanvas instanceof ImageData) {
pixels = tf15.browser.fromPixels(outCanvas); pixels = tf16.browser.fromPixels(outCanvas);
} else if (config3.backend === "webgl" || config3.backend === "humangl") { } else if (config3.backend === "webgl" || config3.backend === "humangl") {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight; tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
pixels = tf15.browser.fromPixels(tempCanvas); pixels = tf16.browser.fromPixels(tempCanvas);
} else { } else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
@ -18277,7 +18361,7 @@ function process3(input, config3) {
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight); const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf15.browser.fromPixels(data); pixels = tf16.browser.fromPixels(data);
} }
const casted = pixels.toFloat(); const casted = pixels.toFloat();
tensor = casted.expandDims(0); tensor = casted.expandDims(0);
@ -18311,7 +18395,7 @@ var options = {
roundRect: 28, roundRect: 28,
drawPoints: false, drawPoints: false,
drawLabels: true, drawLabels: true,
drawBoxes: false, drawBoxes: true,
drawPolygons: true, drawPolygons: true,
fillPolygons: false, fillPolygons: false,
useDepth: true, useDepth: true,
@ -19537,7 +19621,7 @@ var Human = class {
return null; return null;
if (!input) if (!input)
return "input is not defined"; return "input is not defined";
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor)) if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
return "input must be a tensor"; return "input must be a tensor";
try { try {
this.tf.getBackend(); this.tf.getBackend();
@ -19600,7 +19684,7 @@ var Human = class {
}); });
__privateAdd(this, _skipFrame, async (input) => { __privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return true; return false;
const resizeFact = 50; const resizeFact = 50;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
@ -19673,8 +19757,8 @@ var Human = class {
if (!img) if (!img)
return null; return null;
let res; let res;
if (typeof tf16["node"] !== "undefined") { if (typeof tf17["node"] !== "undefined") {
const data = tf16["node"].decodeJpeg(img); const data = tf17["node"].decodeJpeg(img);
const expanded = data.expandDims(0); const expanded = data.expandDims(0);
this.tf.dispose(data); this.tf.dispose(data);
res = await this.detect(expanded, this.config); res = await this.detect(expanded, this.config);
@ -19685,7 +19769,7 @@ var Human = class {
} }
return res; return res;
}); });
this.tf = tf16; this.tf = tf17;
this.draw = draw_exports; this.draw = draw_exports;
this.version = version; this.version = version;
this.config = mergeDeep(config, userConfig); this.config = mergeDeep(config, userConfig);
@ -19707,16 +19791,18 @@ var Human = class {
emotion: null, emotion: null,
embedding: null, embedding: null,
nanodet: null, nanodet: null,
centernet: null,
faceres: null faceres: null
}; };
this.image = (input) => process3(input, this.config); this.image = (input) => process4(input, this.config);
this.classes = { this.classes = {
facemesh: facemesh_exports, facemesh: facemesh_exports,
emotion: emotion_exports, emotion: emotion_exports,
faceres: faceres_exports, faceres: faceres_exports,
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports, body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
hand: handpose_exports, hand: handpose_exports,
nanodet: nanodet_exports nanodet: nanodet_exports,
centernet: centernet_exports
}; };
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
@ -19762,6 +19848,7 @@ var Human = class {
this.models.posenet, this.models.posenet,
this.models.blazepose, this.models.blazepose,
this.models.nanodet, this.models.nanodet,
this.models.centernet,
this.models.faceres this.models.faceres
] = await Promise.all([ ] = await Promise.all([
this.models.face || (this.config.face.enabled ? load2(this.config) : null), this.models.face || (this.config.face.enabled ? load2(this.config) : null),
@ -19769,7 +19856,8 @@ var Human = class {
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null), this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null), this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null) this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
]); ]);
} else { } else {
@ -19783,8 +19871,10 @@ var Human = class {
this.models.posenet = await load5(this.config); this.models.posenet = await load5(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose")) if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
this.models.blazepose = await load7(this.config); this.models.blazepose = await load7(this.config);
if (this.config.object.enabled && !this.models.nanodet) if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
this.models.nanodet = await load8(this.config); this.models.nanodet = await load8(this.config);
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
this.models.centernet = await load9(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
this.models.faceres = await load4(this.config); this.models.faceres = await load4(this.config);
} }
@ -19812,8 +19902,8 @@ var Human = class {
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process5 = process4(input, this.config);
if (!process4 || !process4.tensor) { if (!process5 || !process5.tensor) {
log("could not convert input to tensor"); log("could not convert input to tensor");
resolve({ error: "could not convert input to tensor" }); resolve({ error: "could not convert input to tensor" });
return; return;
@ -19821,7 +19911,7 @@ var Human = class {
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now(); timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor); this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
if (!this.perf.frames) if (!this.perf.frames)
this.perf.frames = 0; this.perf.frames = 0;
if (!this.perf.cached) if (!this.perf.cached)
@ -19837,13 +19927,13 @@ var Human = class {
let objectRes; let objectRes;
let current; let current;
if (this.config.async) { if (this.config.async) {
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
if (this.perf.face) if (this.perf.face)
delete this.perf.face; delete this.perf.face;
} else { } else {
this.state = "run:face"; this.state = "run:face";
timeStamp = now(); timeStamp = now();
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.face = current; this.perf.face = current;
@ -19851,18 +19941,18 @@ var Human = class {
this.analyze("Start Body:"); this.analyze("Start Body:");
if (this.config.async) { if (this.config.async) {
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
if (this.perf.body) if (this.perf.body)
delete this.perf.body; delete this.perf.body;
} else { } else {
this.state = "run:body"; this.state = "run:body";
timeStamp = now(); timeStamp = now();
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.body = current; this.perf.body = current;
@ -19870,13 +19960,13 @@ var Human = class {
this.analyze("End Body:"); this.analyze("End Body:");
this.analyze("Start Hand:"); this.analyze("Start Hand:");
if (this.config.async) { if (this.config.async) {
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
if (this.perf.hand) if (this.perf.hand)
delete this.perf.hand; delete this.perf.hand;
} else { } else {
this.state = "run:hand"; this.state = "run:hand";
timeStamp = now(); timeStamp = now();
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.hand = current; this.perf.hand = current;
@ -19884,13 +19974,19 @@ var Human = class {
this.analyze("End Hand:"); this.analyze("End Hand:");
this.analyze("Start Object:"); this.analyze("Start Object:");
if (this.config.async) { if (this.config.async) {
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
if (this.perf.object) if (this.perf.object)
delete this.perf.object; delete this.perf.object;
} else { } else {
this.state = "run:object"; this.state = "run:object";
timeStamp = now(); timeStamp = now();
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.object = current; this.perf.object = current;
@ -19899,7 +19995,7 @@ var Human = class {
if (this.config.async) { if (this.config.async) {
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]); [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
} }
tf16.dispose(process4.tensor); tf17.dispose(process5.tensor);
let gestureRes = []; let gestureRes = [];
if (this.config.gesture.enabled) { if (this.config.gesture.enabled) {
timeStamp = now(); timeStamp = now();
@ -19918,7 +20014,7 @@ var Human = class {
gesture: gestureRes, gesture: gestureRes,
object: objectRes, object: objectRes,
performance: this.perf, performance: this.perf,
canvas: process4.canvas canvas: process5.canvas
}; };
resolve(result); resolve(result);
}); });

262
dist/human.node-wasm.js vendored
View File

@ -128,7 +128,7 @@ var config = {
debug: true, debug: true,
async: true, async: true,
warmup: "full", warmup: "full",
cacheSensitivity: 4e-3, cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -193,7 +193,7 @@ var config = {
hand: { hand: {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 32,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -207,7 +207,7 @@ var config = {
}, },
object: { object: {
enabled: false, enabled: false,
modelPath: "nanodet.json", modelPath: "mb3-centernet.json",
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
@ -237,7 +237,7 @@ function info() {
} }
// src/human.ts // src/human.ts
var tf16 = __toModule(require_tfjs_esm()); var tf17 = __toModule(require_tfjs_esm());
// src/tfjs/backend.ts // src/tfjs/backend.ts
var tf = __toModule(require_tfjs_esm()); var tf = __toModule(require_tfjs_esm());
@ -339,16 +339,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image12, cropSize) { function cutBoxFromImageAndResize(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image12, boxes, [0], cropSize); return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -482,11 +482,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
return tf3.concat2d([startNormalized, endNormalized], concatAxis); return tf3.concat2d([startNormalized, endNormalized], concatAxis);
} }
var BlazeFaceModel = class { var BlazeFaceModel = class {
constructor(model6, config3) { constructor(model7, config3) {
this.model = model6; this.model = model7;
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]); this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
this.anchors = tf3.tensor2d(this.anchorsData); this.anchors = tf3.tensor2d(this.anchorsData);
this.inputSize = model6.inputs[0].shape[2]; this.inputSize = model7.inputs[0].shape[2];
this.config = config3; this.config = config3;
} }
async getBoundingBoxes(inputImage) { async getBoundingBoxes(inputImage) {
@ -535,12 +535,12 @@ var BlazeFaceModel = class {
} }
}; };
async function load(config3) { async function load(config3) {
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")}); const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
const blazeFace = new BlazeFaceModel(model6, config3); const blazeFace = new BlazeFaceModel(model7, config3);
if (!model6 || !model6.modelUrl) if (!model7 || !model7.modelUrl)
log("load model failed:", config3.face.detector.modelPath); log("load model failed:", config3.face.detector.modelPath);
else if (config3.debug) else if (config3.debug)
log("load model:", model6.modelUrl); log("load model:", model7.modelUrl);
return blazeFace; return blazeFace;
} }
@ -4175,7 +4175,7 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image12, config3, idx, count2) { async function predict2(image13, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
@ -4184,7 +4184,7 @@ async function predict2(image12, config3, idx, count2) {
} }
skipped = 0; skipped = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4267,7 +4267,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image12 = tf7.tidy(() => { const image13 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4276,9 +4276,9 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image12; return image13;
} }
async function predict3(image12, config3, idx, count2) { async function predict3(image13, config3, idx, count2) {
var _a, _b; var _a, _b;
if (!model2) if (!model2)
return null; return null;
@ -4288,7 +4288,7 @@ async function predict3(image12, config3, idx, count2) {
} }
skipped2 = 0; skipped2 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image12); const enhanced = enhance(image13);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4845,16 +4845,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image12, cropSize) { function cutBoxFromImageAndResize2(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image12, boxes, [0], cropSize); return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16665,9 +16665,9 @@ var anchors = [
// src/handpose/handdetector.ts // src/handpose/handdetector.ts
var HandDetector = class { var HandDetector = class {
constructor(model6) { constructor(model7) {
var _a; var _a;
this.model = model6; this.model = model7;
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]); this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
this.anchorsTensor = tf10.tensor2d(this.anchors); this.anchorsTensor = tf10.tensor2d(this.anchors);
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2]; this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
@ -16721,9 +16721,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image12, config3); const predictions = await this.getBoxes(image13, config3);
image12.dispose(); image13.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16868,11 +16868,11 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image12, config3) { async estimateHands(image13, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image12, config3); boxes = await this.handDetector.estimateHandBounds(image13, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.skipFrame) if (config3.skipFrame)
@ -16891,8 +16891,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17102,13 +17102,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image12, config3) { async function predict6(image13, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image12.shape[2], height: image12.shape[1]}; const imgSize = { width: image13.shape[2], height: image13.shape[1] };
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17135,7 +17135,7 @@ async function predict6(image12, config3) {
return [{ score, keypoints }]; return [{ score, keypoints }];
} }
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var nanodet_exports = {}; var nanodet_exports = {};
__export(nanodet_exports, { __export(nanodet_exports, {
load: () => load8, load: () => load8,
@ -17143,7 +17143,7 @@ __export(nanodet_exports, {
}); });
var tf14 = __toModule(require_tfjs_esm()); var tf14 = __toModule(require_tfjs_esm());
// src/nanodet/labels.ts // src/object/labels.ts
var labels = [ var labels = [
{ class: 1, label: "person" }, { class: 1, label: "person" },
{ class: 2, label: "bicycle" }, { class: 2, label: "bicycle" },
@ -17227,7 +17227,7 @@ var labels = [
{ class: 80, label: "toothbrush" } { class: 80, label: "toothbrush" }
]; ];
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var model5; var model5;
var last3 = []; var last3 = [];
var skipped3 = Number.MAX_SAFE_INTEGER; var skipped3 = Number.MAX_SAFE_INTEGER;
@ -17300,7 +17300,7 @@ async function process2(res, inputSize, outputShape, config3) {
}); });
} }
res.forEach((t) => tf14.dispose(t)); res.forEach((t) => tf14.dispose(t));
const nmsBoxes = results.map((a) => a.boxRaw); const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
const nmsScores = results.map((a) => a.score); const nmsScores = results.map((a) => a.score);
let nmsIdx = []; let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) { if (nmsBoxes && nmsBoxes.length > 0) {
@ -17311,7 +17311,7 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image12, config3) { async function predict7(image13, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
@ -17320,8 +17320,8 @@ async function predict7(image12, config3) {
} }
skipped3 = 0; skipped3 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image12.shape[2], image12.shape[1]]; const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17336,6 +17336,90 @@ async function predict7(image12, config3) {
}); });
} }
// src/object/centernet.ts
var centernet_exports = {};
__export(centernet_exports, {
load: () => load9,
predict: () => predict8
});
var tf15 = __toModule(require_tfjs_esm());
var model6;
var last4 = [];
var skipped4 = Number.MAX_SAFE_INTEGER;
async function load9(config3) {
if (!model6) {
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
const inputs = Object.values(model6.modelSignature["inputs"]);
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
if (!model6.inputSize)
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
if (!model6 || !model6.modelUrl)
log("load model failed:", config3.object.modelPath);
else if (config3.debug)
log("load model:", model6.modelUrl);
} else if (config3.debug)
log("cached model:", model6.modelUrl);
return model6;
}
async function process3(res, inputSize, outputShape, config3) {
const results = [];
const detections = res.arraySync();
const squeezeT = tf15.squeeze(res);
res.dispose();
const arr = tf15.split(squeezeT, 6, 1);
squeezeT.dispose();
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose());
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose();
scoresT.dispose();
classesT.dispose();
const nms = nmsT.dataSync();
nmsT.dispose();
for (const id of nms) {
const score = detections[0][id][4];
const classVal = detections[0][id][5];
const label = labels[classVal].label;
const boxRaw = [
detections[0][id][0] / inputSize,
detections[0][id][1] / inputSize,
detections[0][id][2] / inputSize,
detections[0][id][3] / inputSize
];
const box4 = [
Math.trunc(boxRaw[0] * outputShape[0]),
Math.trunc(boxRaw[1] * outputShape[1]),
Math.trunc(boxRaw[2] * outputShape[0]),
Math.trunc(boxRaw[3] * outputShape[1])
];
results.push({ score, class: classVal, label, box: box4, boxRaw });
}
return results;
}
async function predict8(image13, config3) {
if (!model6)
return null;
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
skipped4++;
return last4;
}
skipped4 = 0;
return new Promise(async (resolve) => {
const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
let objectT;
if (config3.object.enabled)
objectT = model6.execute(resize, "tower_0/detections");
resize.dispose();
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
last4 = obj;
resolve(obj);
});
}
// src/gesture/gesture.ts // src/gesture/gesture.ts
var body = (res) => { var body = (res) => {
if (!res) if (!res)
@ -17445,7 +17529,7 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var tf15 = __toModule(require_tfjs_esm()); var tf16 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
function GLProgram(gl, vertexSource, fragmentSource) { function GLProgram(gl, vertexSource, fragmentSource) {
@ -17597,8 +17681,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image12) { this.apply = function(image13) {
_resize(image12.width, image12.height); _resize(image13.width, image13.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17607,7 +17691,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -18158,16 +18242,16 @@ var maxSize = 2048;
var inCanvas; var inCanvas;
var outCanvas; var outCanvas;
var fx; var fx;
function process3(input, config3) { function process4(input, config3) {
let tensor; let tensor;
if (!input) if (!input)
throw new Error("Human: Input is missing"); throw new Error("Human: Input is missing");
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) { if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
throw new Error("Human: Input type is not recognized"); throw new Error("Human: Input type is not recognized");
} }
if (input instanceof tf15.Tensor) { if (input instanceof tf16.Tensor) {
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
tensor = tf15.clone(input); tensor = tf16.clone(input);
else else
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`); throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
} else { } else {
@ -18220,7 +18304,7 @@ function process3(input, config3) {
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width; outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height)) if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height; outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null; fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
} }
if (!fx) if (!fx)
return { tensor: null, canvas: inCanvas }; return { tensor: null, canvas: inCanvas };
@ -18261,16 +18345,16 @@ function process3(input, config3) {
let pixels; let pixels;
if (outCanvas.data) { if (outCanvas.data) {
const shape = [outCanvas.height, outCanvas.width, 3]; const shape = [outCanvas.height, outCanvas.width, 3];
pixels = tf15.tensor3d(outCanvas.data, shape, "int32"); pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
} else if (outCanvas instanceof ImageData) { } else if (outCanvas instanceof ImageData) {
pixels = tf15.browser.fromPixels(outCanvas); pixels = tf16.browser.fromPixels(outCanvas);
} else if (config3.backend === "webgl" || config3.backend === "humangl") { } else if (config3.backend === "webgl" || config3.backend === "humangl") {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight; tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
pixels = tf15.browser.fromPixels(tempCanvas); pixels = tf16.browser.fromPixels(tempCanvas);
} else { } else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
@ -18278,7 +18362,7 @@ function process3(input, config3) {
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight); const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf15.browser.fromPixels(data); pixels = tf16.browser.fromPixels(data);
} }
const casted = pixels.toFloat(); const casted = pixels.toFloat();
tensor = casted.expandDims(0); tensor = casted.expandDims(0);
@ -18312,7 +18396,7 @@ var options = {
roundRect: 28, roundRect: 28,
drawPoints: false, drawPoints: false,
drawLabels: true, drawLabels: true,
drawBoxes: false, drawBoxes: true,
drawPolygons: true, drawPolygons: true,
fillPolygons: false, fillPolygons: false,
useDepth: true, useDepth: true,
@ -19538,7 +19622,7 @@ var Human = class {
return null; return null;
if (!input) if (!input)
return "input is not defined"; return "input is not defined";
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor)) if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
return "input must be a tensor"; return "input must be a tensor";
try { try {
this.tf.getBackend(); this.tf.getBackend();
@ -19601,7 +19685,7 @@ var Human = class {
}); });
__privateAdd(this, _skipFrame, async (input) => { __privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return true; return false;
const resizeFact = 50; const resizeFact = 50;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
@ -19674,8 +19758,8 @@ var Human = class {
if (!img) if (!img)
return null; return null;
let res; let res;
if (typeof tf16["node"] !== "undefined") { if (typeof tf17["node"] !== "undefined") {
const data = tf16["node"].decodeJpeg(img); const data = tf17["node"].decodeJpeg(img);
const expanded = data.expandDims(0); const expanded = data.expandDims(0);
this.tf.dispose(data); this.tf.dispose(data);
res = await this.detect(expanded, this.config); res = await this.detect(expanded, this.config);
@ -19686,7 +19770,7 @@ var Human = class {
} }
return res; return res;
}); });
this.tf = tf16; this.tf = tf17;
this.draw = draw_exports; this.draw = draw_exports;
this.version = version; this.version = version;
this.config = mergeDeep(config, userConfig); this.config = mergeDeep(config, userConfig);
@ -19708,16 +19792,18 @@ var Human = class {
emotion: null, emotion: null,
embedding: null, embedding: null,
nanodet: null, nanodet: null,
centernet: null,
faceres: null faceres: null
}; };
this.image = (input) => process3(input, this.config); this.image = (input) => process4(input, this.config);
this.classes = { this.classes = {
facemesh: facemesh_exports, facemesh: facemesh_exports,
emotion: emotion_exports, emotion: emotion_exports,
faceres: faceres_exports, faceres: faceres_exports,
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports, body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
hand: handpose_exports, hand: handpose_exports,
nanodet: nanodet_exports nanodet: nanodet_exports,
centernet: centernet_exports
}; };
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
@ -19763,6 +19849,7 @@ var Human = class {
this.models.posenet, this.models.posenet,
this.models.blazepose, this.models.blazepose,
this.models.nanodet, this.models.nanodet,
this.models.centernet,
this.models.faceres this.models.faceres
] = await Promise.all([ ] = await Promise.all([
this.models.face || (this.config.face.enabled ? load2(this.config) : null), this.models.face || (this.config.face.enabled ? load2(this.config) : null),
@ -19770,7 +19857,8 @@ var Human = class {
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null), this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null), this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null) this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
]); ]);
} else { } else {
@ -19784,8 +19872,10 @@ var Human = class {
this.models.posenet = await load5(this.config); this.models.posenet = await load5(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose")) if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
this.models.blazepose = await load7(this.config); this.models.blazepose = await load7(this.config);
if (this.config.object.enabled && !this.models.nanodet) if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
this.models.nanodet = await load8(this.config); this.models.nanodet = await load8(this.config);
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
this.models.centernet = await load9(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
this.models.faceres = await load4(this.config); this.models.faceres = await load4(this.config);
} }
@ -19813,8 +19903,8 @@ var Human = class {
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process5 = process4(input, this.config);
if (!process4 || !process4.tensor) { if (!process5 || !process5.tensor) {
log("could not convert input to tensor"); log("could not convert input to tensor");
resolve({ error: "could not convert input to tensor" }); resolve({ error: "could not convert input to tensor" });
return; return;
@ -19822,7 +19912,7 @@ var Human = class {
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now(); timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor); this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
if (!this.perf.frames) if (!this.perf.frames)
this.perf.frames = 0; this.perf.frames = 0;
if (!this.perf.cached) if (!this.perf.cached)
@ -19838,13 +19928,13 @@ var Human = class {
let objectRes; let objectRes;
let current; let current;
if (this.config.async) { if (this.config.async) {
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
if (this.perf.face) if (this.perf.face)
delete this.perf.face; delete this.perf.face;
} else { } else {
this.state = "run:face"; this.state = "run:face";
timeStamp = now(); timeStamp = now();
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.face = current; this.perf.face = current;
@ -19852,18 +19942,18 @@ var Human = class {
this.analyze("Start Body:"); this.analyze("Start Body:");
if (this.config.async) { if (this.config.async) {
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
if (this.perf.body) if (this.perf.body)
delete this.perf.body; delete this.perf.body;
} else { } else {
this.state = "run:body"; this.state = "run:body";
timeStamp = now(); timeStamp = now();
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.body = current; this.perf.body = current;
@ -19871,13 +19961,13 @@ var Human = class {
this.analyze("End Body:"); this.analyze("End Body:");
this.analyze("Start Hand:"); this.analyze("Start Hand:");
if (this.config.async) { if (this.config.async) {
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
if (this.perf.hand) if (this.perf.hand)
delete this.perf.hand; delete this.perf.hand;
} else { } else {
this.state = "run:hand"; this.state = "run:hand";
timeStamp = now(); timeStamp = now();
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.hand = current; this.perf.hand = current;
@ -19885,13 +19975,19 @@ var Human = class {
this.analyze("End Hand:"); this.analyze("End Hand:");
this.analyze("Start Object:"); this.analyze("Start Object:");
if (this.config.async) { if (this.config.async) {
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
if (this.perf.object) if (this.perf.object)
delete this.perf.object; delete this.perf.object;
} else { } else {
this.state = "run:object"; this.state = "run:object";
timeStamp = now(); timeStamp = now();
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.object = current; this.perf.object = current;
@ -19900,7 +19996,7 @@ var Human = class {
if (this.config.async) { if (this.config.async) {
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]); [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
} }
tf16.dispose(process4.tensor); tf17.dispose(process5.tensor);
let gestureRes = []; let gestureRes = [];
if (this.config.gesture.enabled) { if (this.config.gesture.enabled) {
timeStamp = now(); timeStamp = now();
@ -19919,7 +20015,7 @@ var Human = class {
gesture: gestureRes, gesture: gestureRes,
object: objectRes, object: objectRes,
performance: this.perf, performance: this.perf,
canvas: process4.canvas canvas: process5.canvas
}; };
resolve(result); resolve(result);
}); });

262
dist/human.node.js vendored
View File

@ -127,7 +127,7 @@ var config = {
debug: true, debug: true,
async: true, async: true,
warmup: "full", warmup: "full",
cacheSensitivity: 4e-3, cacheSensitivity: 5e-3,
filter: { filter: {
enabled: true, enabled: true,
width: 0, width: 0,
@ -192,7 +192,7 @@ var config = {
hand: { hand: {
enabled: true, enabled: true,
rotation: false, rotation: false,
skipFrames: 12, skipFrames: 32,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -206,7 +206,7 @@ var config = {
}, },
object: { object: {
enabled: false, enabled: false,
modelPath: "nanodet.json", modelPath: "mb3-centernet.json",
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
@ -236,7 +236,7 @@ function info() {
} }
// src/human.ts // src/human.ts
var tf16 = __toModule(require_tfjs_esm()); var tf17 = __toModule(require_tfjs_esm());
// src/tfjs/backend.ts // src/tfjs/backend.ts
var tf = __toModule(require_tfjs_esm()); var tf = __toModule(require_tfjs_esm());
@ -338,16 +338,16 @@ function getBoxCenter(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize(box4, image12, cropSize) { function cutBoxFromImageAndResize(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf2.image.cropAndResize(image12, boxes, [0], cropSize); return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function enlargeBox(box4, factor = 1.5) { function enlargeBox(box4, factor = 1.5) {
const center = getBoxCenter(box4); const center = getBoxCenter(box4);
@ -481,11 +481,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
return tf3.concat2d([startNormalized, endNormalized], concatAxis); return tf3.concat2d([startNormalized, endNormalized], concatAxis);
} }
var BlazeFaceModel = class { var BlazeFaceModel = class {
constructor(model6, config3) { constructor(model7, config3) {
this.model = model6; this.model = model7;
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]); this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
this.anchors = tf3.tensor2d(this.anchorsData); this.anchors = tf3.tensor2d(this.anchorsData);
this.inputSize = model6.inputs[0].shape[2]; this.inputSize = model7.inputs[0].shape[2];
this.config = config3; this.config = config3;
} }
async getBoundingBoxes(inputImage) { async getBoundingBoxes(inputImage) {
@ -534,12 +534,12 @@ var BlazeFaceModel = class {
} }
}; };
async function load(config3) { async function load(config3) {
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")}); const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
const blazeFace = new BlazeFaceModel(model6, config3); const blazeFace = new BlazeFaceModel(model7, config3);
if (!model6 || !model6.modelUrl) if (!model7 || !model7.modelUrl)
log("load model failed:", config3.face.detector.modelPath); log("load model failed:", config3.face.detector.modelPath);
else if (config3.debug) else if (config3.debug)
log("load model:", model6.modelUrl); log("load model:", model7.modelUrl);
return blazeFace; return blazeFace;
} }
@ -4174,7 +4174,7 @@ async function load3(config3) {
log("cached model:", model.modelUrl); log("cached model:", model.modelUrl);
return model; return model;
} }
async function predict2(image12, config3, idx, count2) { async function predict2(image13, config3, idx, count2) {
if (!model) if (!model)
return null; return null;
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) { if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
@ -4183,7 +4183,7 @@ async function predict2(image12, config3, idx, count2) {
} }
skipped = 0; skipped = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
const [red, green, blue] = tf6.split(resize, 3, 3); const [red, green, blue] = tf6.split(resize, 3, 3);
resize.dispose(); resize.dispose();
const redNorm = tf6.mul(red, rgb[0]); const redNorm = tf6.mul(red, rgb[0]);
@ -4266,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
return best; return best;
} }
function enhance(input) { function enhance(input) {
const image12 = tf7.tidy(() => { const image13 = tf7.tidy(() => {
const tensor = input.image || input.tensor || input; const tensor = input.image || input.tensor || input;
if (!(tensor instanceof tf7.Tensor)) if (!(tensor instanceof tf7.Tensor))
return null; return null;
@ -4275,9 +4275,9 @@ function enhance(input) {
const norm = crop.mul(255); const norm = crop.mul(255);
return norm; return norm;
}); });
return image12; return image13;
} }
async function predict3(image12, config3, idx, count2) { async function predict3(image13, config3, idx, count2) {
var _a, _b; var _a, _b;
if (!model2) if (!model2)
return null; return null;
@ -4287,7 +4287,7 @@ async function predict3(image12, config3, idx, count2) {
} }
skipped2 = 0; skipped2 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image12); const enhanced = enhance(image13);
let resT; let resT;
const obj = { const obj = {
age: 0, age: 0,
@ -4844,16 +4844,16 @@ function getBoxCenter2(box4) {
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2 box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
]; ];
} }
function cutBoxFromImageAndResize2(box4, image12, cropSize) { function cutBoxFromImageAndResize2(box4, image13, cropSize) {
const h = image12.shape[1]; const h = image13.shape[1];
const w = image12.shape[2]; const w = image13.shape[2];
const boxes = [[ const boxes = [[
box4.startPoint[1] / h, box4.startPoint[1] / h,
box4.startPoint[0] / w, box4.startPoint[0] / w,
box4.endPoint[1] / h, box4.endPoint[1] / h,
box4.endPoint[0] / w box4.endPoint[0] / w
]]; ]];
return tf9.image.cropAndResize(image12, boxes, [0], cropSize); return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
} }
function scaleBoxCoordinates2(box4, factor) { function scaleBoxCoordinates2(box4, factor) {
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]]; const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
@ -16664,9 +16664,9 @@ var anchors = [
// src/handpose/handdetector.ts // src/handpose/handdetector.ts
var HandDetector = class { var HandDetector = class {
constructor(model6) { constructor(model7) {
var _a; var _a;
this.model = model6; this.model = model7;
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]); this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
this.anchorsTensor = tf10.tensor2d(this.anchors); this.anchorsTensor = tf10.tensor2d(this.anchors);
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2]; this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
@ -16720,9 +16720,9 @@ var HandDetector = class {
async estimateHandBounds(input, config3) { async estimateHandBounds(input, config3) {
const inputHeight = input.shape[1]; const inputHeight = input.shape[1];
const inputWidth = input.shape[2]; const inputWidth = input.shape[2];
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoxes(image12, config3); const predictions = await this.getBoxes(image13, config3);
image12.dispose(); image13.dispose();
const hands = []; const hands = [];
if (!predictions || predictions.length === 0) if (!predictions || predictions.length === 0)
return hands; return hands;
@ -16867,11 +16867,11 @@ var HandPipeline = class {
coord[2] coord[2]
]); ]);
} }
async estimateHands(image12, config3) { async estimateHands(image13, config3) {
let useFreshBox = false; let useFreshBox = false;
let boxes; let boxes;
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) { if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image12, config3); boxes = await this.handDetector.estimateHandBounds(image13, config3);
this.skipped = 0; this.skipped = 0;
} }
if (config3.skipFrame) if (config3.skipFrame)
@ -16890,8 +16890,8 @@ var HandPipeline = class {
if (config3.hand.landmarks) { if (config3.hand.landmarks) {
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
const palmCenter = getBoxCenter2(currentBox); const palmCenter = getBoxCenter2(currentBox);
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]]; const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone(); const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter); const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]); const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
@ -17101,13 +17101,13 @@ async function load7(config3) {
log("cached model:", model4.modelUrl); log("cached model:", model4.modelUrl);
return model4; return model4;
} }
async function predict6(image12, config3) { async function predict6(image13, config3) {
if (!model4) if (!model4)
return null; return null;
if (!config3.body.enabled) if (!config3.body.enabled)
return null; return null;
const imgSize = {width: image12.shape[2], height: image12.shape[1]}; const imgSize = { width: image13.shape[2], height: image13.shape[1] };
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false); const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
const normalize = tf13.div(resize, [255]); const normalize = tf13.div(resize, [255]);
resize.dispose(); resize.dispose();
const resT = await model4.predict(normalize); const resT = await model4.predict(normalize);
@ -17134,7 +17134,7 @@ async function predict6(image12, config3) {
return [{ score, keypoints }]; return [{ score, keypoints }];
} }
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var nanodet_exports = {}; var nanodet_exports = {};
__export(nanodet_exports, { __export(nanodet_exports, {
load: () => load8, load: () => load8,
@ -17142,7 +17142,7 @@ __export(nanodet_exports, {
}); });
var tf14 = __toModule(require_tfjs_esm()); var tf14 = __toModule(require_tfjs_esm());
// src/nanodet/labels.ts // src/object/labels.ts
var labels = [ var labels = [
{ class: 1, label: "person" }, { class: 1, label: "person" },
{ class: 2, label: "bicycle" }, { class: 2, label: "bicycle" },
@ -17226,7 +17226,7 @@ var labels = [
{ class: 80, label: "toothbrush" } { class: 80, label: "toothbrush" }
]; ];
// src/nanodet/nanodet.ts // src/object/nanodet.ts
var model5; var model5;
var last3 = []; var last3 = [];
var skipped3 = Number.MAX_SAFE_INTEGER; var skipped3 = Number.MAX_SAFE_INTEGER;
@ -17299,7 +17299,7 @@ async function process2(res, inputSize, outputShape, config3) {
}); });
} }
res.forEach((t) => tf14.dispose(t)); res.forEach((t) => tf14.dispose(t));
const nmsBoxes = results.map((a) => a.boxRaw); const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
const nmsScores = results.map((a) => a.score); const nmsScores = results.map((a) => a.score);
let nmsIdx = []; let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) { if (nmsBoxes && nmsBoxes.length > 0) {
@ -17310,7 +17310,7 @@ async function process2(res, inputSize, outputShape, config3) {
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score); results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
return results; return results;
} }
async function predict7(image12, config3) { async function predict7(image13, config3) {
if (!model5) if (!model5)
return null; return null;
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) { if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
@ -17319,8 +17319,8 @@ async function predict7(image12, config3) {
} }
skipped3 = 0; skipped3 = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const outputSize = [image12.shape[2], image12.shape[1]]; const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false); const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
const norm = resize.div(255); const norm = resize.div(255);
const transpose = norm.transpose([0, 3, 1, 2]); const transpose = norm.transpose([0, 3, 1, 2]);
norm.dispose(); norm.dispose();
@ -17335,6 +17335,90 @@ async function predict7(image12, config3) {
}); });
} }
// src/object/centernet.ts
var centernet_exports = {};
__export(centernet_exports, {
load: () => load9,
predict: () => predict8
});
var tf15 = __toModule(require_tfjs_esm());
var model6;
var last4 = [];
var skipped4 = Number.MAX_SAFE_INTEGER;
async function load9(config3) {
if (!model6) {
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
const inputs = Object.values(model6.modelSignature["inputs"]);
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
if (!model6.inputSize)
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
if (!model6 || !model6.modelUrl)
log("load model failed:", config3.object.modelPath);
else if (config3.debug)
log("load model:", model6.modelUrl);
} else if (config3.debug)
log("cached model:", model6.modelUrl);
return model6;
}
async function process3(res, inputSize, outputShape, config3) {
const results = [];
const detections = res.arraySync();
const squeezeT = tf15.squeeze(res);
res.dispose();
const arr = tf15.split(squeezeT, 6, 1);
squeezeT.dispose();
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose());
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
boxesT.dispose();
scoresT.dispose();
classesT.dispose();
const nms = nmsT.dataSync();
nmsT.dispose();
for (const id of nms) {
const score = detections[0][id][4];
const classVal = detections[0][id][5];
const label = labels[classVal].label;
const boxRaw = [
detections[0][id][0] / inputSize,
detections[0][id][1] / inputSize,
detections[0][id][2] / inputSize,
detections[0][id][3] / inputSize
];
const box4 = [
Math.trunc(boxRaw[0] * outputShape[0]),
Math.trunc(boxRaw[1] * outputShape[1]),
Math.trunc(boxRaw[2] * outputShape[0]),
Math.trunc(boxRaw[3] * outputShape[1])
];
results.push({ score, class: classVal, label, box: box4, boxRaw });
}
return results;
}
async function predict8(image13, config3) {
if (!model6)
return null;
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
skipped4++;
return last4;
}
skipped4 = 0;
return new Promise(async (resolve) => {
const outputSize = [image13.shape[2], image13.shape[1]];
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
let objectT;
if (config3.object.enabled)
objectT = model6.execute(resize, "tower_0/detections");
resize.dispose();
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
last4 = obj;
resolve(obj);
});
}
// src/gesture/gesture.ts // src/gesture/gesture.ts
var body = (res) => { var body = (res) => {
if (!res) if (!res)
@ -17444,7 +17528,7 @@ var hand = (res) => {
}; };
// src/image/image.ts // src/image/image.ts
var tf15 = __toModule(require_tfjs_esm()); var tf16 = __toModule(require_tfjs_esm());
// src/image/imagefx.js // src/image/imagefx.js
function GLProgram(gl, vertexSource, fragmentSource) { function GLProgram(gl, vertexSource, fragmentSource) {
@ -17596,8 +17680,8 @@ function GLImageFilter(params) {
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1); gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function(image12) { this.apply = function(image13) {
_resize(image12.width, image12.height); _resize(image13.width, image13.height);
_drawCount = 0; _drawCount = 0;
if (!_sourceTexture) if (!_sourceTexture)
_sourceTexture = gl.createTexture(); _sourceTexture = gl.createTexture();
@ -17606,7 +17690,7 @@ function GLImageFilter(params) {
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
if (_filterChain.length === 0) { if (_filterChain.length === 0) {
_draw(); _draw();
return _canvas; return _canvas;
@ -18157,16 +18241,16 @@ var maxSize = 2048;
var inCanvas; var inCanvas;
var outCanvas; var outCanvas;
var fx; var fx;
function process3(input, config3) { function process4(input, config3) {
let tensor; let tensor;
if (!input) if (!input)
throw new Error("Human: Input is missing"); throw new Error("Human: Input is missing");
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) { if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
throw new Error("Human: Input type is not recognized"); throw new Error("Human: Input type is not recognized");
} }
if (input instanceof tf15.Tensor) { if (input instanceof tf16.Tensor) {
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3) if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
tensor = tf15.clone(input); tensor = tf16.clone(input);
else else
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`); throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
} else { } else {
@ -18219,7 +18303,7 @@ function process3(input, config3) {
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width; outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height)) if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height; outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null; fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
} }
if (!fx) if (!fx)
return { tensor: null, canvas: inCanvas }; return { tensor: null, canvas: inCanvas };
@ -18260,16 +18344,16 @@ function process3(input, config3) {
let pixels; let pixels;
if (outCanvas.data) { if (outCanvas.data) {
const shape = [outCanvas.height, outCanvas.width, 3]; const shape = [outCanvas.height, outCanvas.width, 3];
pixels = tf15.tensor3d(outCanvas.data, shape, "int32"); pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
} else if (outCanvas instanceof ImageData) { } else if (outCanvas instanceof ImageData) {
pixels = tf15.browser.fromPixels(outCanvas); pixels = tf16.browser.fromPixels(outCanvas);
} else if (config3.backend === "webgl" || config3.backend === "humangl") { } else if (config3.backend === "webgl" || config3.backend === "humangl") {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
tempCanvas.height = targetHeight; tempCanvas.height = targetHeight;
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
pixels = tf15.browser.fromPixels(tempCanvas); pixels = tf16.browser.fromPixels(tempCanvas);
} else { } else {
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas"); const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
tempCanvas.width = targetWidth; tempCanvas.width = targetWidth;
@ -18277,7 +18361,7 @@ function process3(input, config3) {
const tempCtx = tempCanvas.getContext("2d"); const tempCtx = tempCanvas.getContext("2d");
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0); tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight); const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
pixels = tf15.browser.fromPixels(data); pixels = tf16.browser.fromPixels(data);
} }
const casted = pixels.toFloat(); const casted = pixels.toFloat();
tensor = casted.expandDims(0); tensor = casted.expandDims(0);
@ -18311,7 +18395,7 @@ var options = {
roundRect: 28, roundRect: 28,
drawPoints: false, drawPoints: false,
drawLabels: true, drawLabels: true,
drawBoxes: false, drawBoxes: true,
drawPolygons: true, drawPolygons: true,
fillPolygons: false, fillPolygons: false,
useDepth: true, useDepth: true,
@ -19537,7 +19621,7 @@ var Human = class {
return null; return null;
if (!input) if (!input)
return "input is not defined"; return "input is not defined";
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor)) if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
return "input must be a tensor"; return "input must be a tensor";
try { try {
this.tf.getBackend(); this.tf.getBackend();
@ -19600,7 +19684,7 @@ var Human = class {
}); });
__privateAdd(this, _skipFrame, async (input) => { __privateAdd(this, _skipFrame, async (input) => {
if (this.config.cacheSensitivity === 0) if (this.config.cacheSensitivity === 0)
return true; return false;
const resizeFact = 50; const resizeFact = 50;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
@ -19673,8 +19757,8 @@ var Human = class {
if (!img) if (!img)
return null; return null;
let res; let res;
if (typeof tf16["node"] !== "undefined") { if (typeof tf17["node"] !== "undefined") {
const data = tf16["node"].decodeJpeg(img); const data = tf17["node"].decodeJpeg(img);
const expanded = data.expandDims(0); const expanded = data.expandDims(0);
this.tf.dispose(data); this.tf.dispose(data);
res = await this.detect(expanded, this.config); res = await this.detect(expanded, this.config);
@ -19685,7 +19769,7 @@ var Human = class {
} }
return res; return res;
}); });
this.tf = tf16; this.tf = tf17;
this.draw = draw_exports; this.draw = draw_exports;
this.version = version; this.version = version;
this.config = mergeDeep(config, userConfig); this.config = mergeDeep(config, userConfig);
@ -19707,16 +19791,18 @@ var Human = class {
emotion: null, emotion: null,
embedding: null, embedding: null,
nanodet: null, nanodet: null,
centernet: null,
faceres: null faceres: null
}; };
this.image = (input) => process3(input, this.config); this.image = (input) => process4(input, this.config);
this.classes = { this.classes = {
facemesh: facemesh_exports, facemesh: facemesh_exports,
emotion: emotion_exports, emotion: emotion_exports,
faceres: faceres_exports, faceres: faceres_exports,
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports, body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
hand: handpose_exports, hand: handpose_exports,
nanodet: nanodet_exports nanodet: nanodet_exports,
centernet: centernet_exports
}; };
this.faceTriangulation = triangulation; this.faceTriangulation = triangulation;
this.faceUVMap = uvmap; this.faceUVMap = uvmap;
@ -19762,6 +19848,7 @@ var Human = class {
this.models.posenet, this.models.posenet,
this.models.blazepose, this.models.blazepose,
this.models.nanodet, this.models.nanodet,
this.models.centernet,
this.models.faceres this.models.faceres
] = await Promise.all([ ] = await Promise.all([
this.models.face || (this.config.face.enabled ? load2(this.config) : null), this.models.face || (this.config.face.enabled ? load2(this.config) : null),
@ -19769,7 +19856,8 @@ var Human = class {
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null), this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null), this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null) this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
]); ]);
} else { } else {
@ -19783,8 +19871,10 @@ var Human = class {
this.models.posenet = await load5(this.config); this.models.posenet = await load5(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose")) if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
this.models.blazepose = await load7(this.config); this.models.blazepose = await load7(this.config);
if (this.config.object.enabled && !this.models.nanodet) if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
this.models.nanodet = await load8(this.config); this.models.nanodet = await load8(this.config);
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
this.models.centernet = await load9(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
this.models.faceres = await load4(this.config); this.models.faceres = await load4(this.config);
} }
@ -19812,8 +19902,8 @@ var Human = class {
await __privateGet(this, _checkBackend).call(this); await __privateGet(this, _checkBackend).call(this);
await this.load(); await this.load();
timeStamp = now(); timeStamp = now();
const process4 = process3(input, this.config); const process5 = process4(input, this.config);
if (!process4 || !process4.tensor) { if (!process5 || !process5.tensor) {
log("could not convert input to tensor"); log("could not convert input to tensor");
resolve({ error: "could not convert input to tensor" }); resolve({ error: "could not convert input to tensor" });
return; return;
@ -19821,7 +19911,7 @@ var Human = class {
this.perf.image = Math.trunc(now() - timeStamp); this.perf.image = Math.trunc(now() - timeStamp);
this.analyze("Get Image:"); this.analyze("Get Image:");
timeStamp = now(); timeStamp = now();
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor); this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
if (!this.perf.frames) if (!this.perf.frames)
this.perf.frames = 0; this.perf.frames = 0;
if (!this.perf.cached) if (!this.perf.cached)
@ -19837,13 +19927,13 @@ var Human = class {
let objectRes; let objectRes;
let current; let current;
if (this.config.async) { if (this.config.async) {
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
if (this.perf.face) if (this.perf.face)
delete this.perf.face; delete this.perf.face;
} else { } else {
this.state = "run:face"; this.state = "run:face";
timeStamp = now(); timeStamp = now();
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : []; faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.face = current; this.perf.face = current;
@ -19851,18 +19941,18 @@ var Human = class {
this.analyze("Start Body:"); this.analyze("Start Body:");
if (this.config.async) { if (this.config.async) {
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
if (this.perf.body) if (this.perf.body)
delete this.perf.body; delete this.perf.body;
} else { } else {
this.state = "run:body"; this.state = "run:body";
timeStamp = now(); timeStamp = now();
if (this.config.body.modelPath.includes("posenet")) if (this.config.body.modelPath.includes("posenet"))
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
else if (this.config.body.modelPath.includes("blazepose")) else if (this.config.body.modelPath.includes("blazepose"))
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : []; bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.body = current; this.perf.body = current;
@ -19870,13 +19960,13 @@ var Human = class {
this.analyze("End Body:"); this.analyze("End Body:");
this.analyze("Start Hand:"); this.analyze("Start Hand:");
if (this.config.async) { if (this.config.async) {
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
if (this.perf.hand) if (this.perf.hand)
delete this.perf.hand; delete this.perf.hand;
} else { } else {
this.state = "run:hand"; this.state = "run:hand";
timeStamp = now(); timeStamp = now();
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : []; handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.hand = current; this.perf.hand = current;
@ -19884,13 +19974,19 @@ var Human = class {
this.analyze("End Hand:"); this.analyze("End Hand:");
this.analyze("Start Object:"); this.analyze("Start Object:");
if (this.config.async) { if (this.config.async) {
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
if (this.perf.object) if (this.perf.object)
delete this.perf.object; delete this.perf.object;
} else { } else {
this.state = "run:object"; this.state = "run:object";
timeStamp = now(); timeStamp = now();
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : []; if (this.config.object.modelPath.includes("nanodet"))
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
else if (this.config.object.modelPath.includes("centernet"))
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) if (current > 0)
this.perf.object = current; this.perf.object = current;
@ -19899,7 +19995,7 @@ var Human = class {
if (this.config.async) { if (this.config.async) {
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]); [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
} }
tf16.dispose(process4.tensor); tf17.dispose(process5.tensor);
let gestureRes = []; let gestureRes = [];
if (this.config.gesture.enabled) { if (this.config.gesture.enabled) {
timeStamp = now(); timeStamp = now();
@ -19918,7 +20014,7 @@ var Human = class {
gesture: gestureRes, gesture: gestureRes,
object: objectRes, object: objectRes,
performance: this.perf, performance: this.perf,
canvas: process4.canvas canvas: process5.canvas
}; };
resolve(result); resolve(result);
}); });

59239
dist/tfjs.esm.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/tfjs.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

BIN
models/mb3-centernet.bin Normal file
View File

Binary file not shown.

577
models/mb3-centernet.json Normal file
View File

File diff suppressed because one or more lines are too long

2
package.json
View File

@ -68,7 +68,7 @@
"canvas": "^2.8.0", "canvas": "^2.8.0",
"chokidar": "^3.5.1", "chokidar": "^3.5.1",
"dayjs": "^1.10.4", "dayjs": "^1.10.4",
"esbuild": "^0.12.0", "esbuild": "^0.12.1",
"eslint": "^7.26.0", "eslint": "^7.26.0",
"eslint-config-airbnb-base": "^14.2.1", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.23.2", "eslint-plugin-import": "^2.23.2",

3
src/config.ts
View File

@ -319,7 +319,8 @@ const config: Config = {
object: { object: {
enabled: false, enabled: false,
modelPath: 'nanodet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath modelPath: 'mb3-centernet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath
// can be 'mb3-centernet' or 'nanodet'
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
maxDetected: 10, // maximum number of objects detected in the input maxDetected: 10, // maximum number of objects detected in the input

2
src/draw/draw.ts
View File

@ -54,7 +54,7 @@ export const options: DrawOptions = {
roundRect: <number>28, roundRect: <number>28,
drawPoints: <Boolean>false, drawPoints: <Boolean>false,
drawLabels: <Boolean>true, drawLabels: <Boolean>true,
drawBoxes: <Boolean>false, drawBoxes: <Boolean>true,
drawPolygons: <Boolean>true, drawPolygons: <Boolean>true,
fillPolygons: <Boolean>false, fillPolygons: <Boolean>false,
useDepth: <Boolean>true, useDepth: <Boolean>true,

30
src/human.ts
View File

@ -11,7 +11,8 @@ import * as emotion from './emotion/emotion';
import * as posenet from './posenet/posenet'; import * as posenet from './posenet/posenet';
import * as handpose from './handpose/handpose'; import * as handpose from './handpose/handpose';
import * as blazepose from './blazepose/blazepose'; import * as blazepose from './blazepose/blazepose';
import * as nanodet from './nanodet/nanodet'; import * as nanodet from './object/nanodet';
import * as centernet from './object/centernet';
import * as gesture from './gesture/gesture'; import * as gesture from './gesture/gesture';
import * as image from './image/image'; import * as image from './image/image';
import * as draw from './draw/draw'; import * as draw from './draw/draw';
@ -93,6 +94,7 @@ export class Human {
emotion: Model | null, emotion: Model | null,
embedding: Model | null, embedding: Model | null,
nanodet: Model | null, nanodet: Model | null,
centernet: Model | null,
faceres: Model | null, faceres: Model | null,
}; };
/** Internal: Currently loaded classes */ /** Internal: Currently loaded classes */
@ -102,6 +104,7 @@ export class Human {
body: typeof posenet | typeof blazepose; body: typeof posenet | typeof blazepose;
hand: typeof handpose; hand: typeof handpose;
nanodet: typeof nanodet; nanodet: typeof nanodet;
centernet: typeof centernet;
faceres: typeof faceres; faceres: typeof faceres;
}; };
/** Face triangualtion array of 468 points, used for triangle references between points */ /** Face triangualtion array of 468 points, used for triangle references between points */
@ -148,6 +151,7 @@ export class Human {
emotion: null, emotion: null,
embedding: null, embedding: null,
nanodet: null, nanodet: null,
centernet: null,
faceres: null, faceres: null,
}; };
// export access to image processing // export access to image processing
@ -161,6 +165,7 @@ export class Human {
body: this.config.body.modelPath.includes('posenet') ? posenet : blazepose, body: this.config.body.modelPath.includes('posenet') ? posenet : blazepose,
hand: handpose, hand: handpose,
nanodet, nanodet,
centernet,
}; };
this.faceTriangulation = facemesh.triangulation; this.faceTriangulation = facemesh.triangulation;
this.faceUVMap = facemesh.uvmap; this.faceUVMap = facemesh.uvmap;
@ -231,7 +236,7 @@ export class Human {
const timeStamp = now(); const timeStamp = now();
if (userConfig) this.config = mergeDeep(this.config, userConfig); if (userConfig) this.config = mergeDeep(this.config, userConfig);
if (this.#firstRun) { if (this.#firstRun) { // print version info on first run and check for correct backend setup
if (this.config.debug) log(`version: ${this.version}`); if (this.config.debug) log(`version: ${this.version}`);
if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`); if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`);
if (this.config.debug) log('platform:', this.sysinfo.platform); if (this.config.debug) log('platform:', this.sysinfo.platform);
@ -243,7 +248,7 @@ export class Human {
if (this.config.debug) log('tf flags:', this.tf.ENV.flags); if (this.config.debug) log('tf flags:', this.tf.ENV.flags);
} }
} }
if (this.config.async) { if (this.config.async) { // load models concurrently
[ [
this.models.face, this.models.face,
this.models.emotion, this.models.emotion,
@ -251,6 +256,7 @@ export class Human {
this.models.posenet, this.models.posenet,
this.models.blazepose, this.models.blazepose,
this.models.nanodet, this.models.nanodet,
this.models.centernet,
this.models.faceres, this.models.faceres,
] = await Promise.all([ ] = await Promise.all([
this.models.face || (this.config.face.enabled ? facemesh.load(this.config) : null), this.models.face || (this.config.face.enabled ? facemesh.load(this.config) : null),
@ -258,20 +264,22 @@ export class Human {
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null), this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null), this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null), this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null), this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes('nanodet') ? nanodet.load(this.config) : null),
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes('centernet') ? centernet.load(this.config) : null),
this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null), this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
]); ]);
} else { } else { // load models sequentially
if (this.config.face.enabled && !this.models.face) this.models.face = await facemesh.load(this.config); if (this.config.face.enabled && !this.models.face) this.models.face = await facemesh.load(this.config);
if (this.config.face.enabled && this.config.face.emotion.enabled && !this.models.emotion) this.models.emotion = await emotion.load(this.config); if (this.config.face.enabled && this.config.face.emotion.enabled && !this.models.emotion) this.models.emotion = await emotion.load(this.config);
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config); if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config); if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config); if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config); if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes('nanodet')) this.models.nanodet = await nanodet.load(this.config);
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes('centernet')) this.models.centernet = await centernet.load(this.config);
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config); if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
} }
if (this.#firstRun) { if (this.#firstRun) { // print memory stats on first run
if (this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors'); if (this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors');
this.#firstRun = false; this.#firstRun = false;
} }
@ -343,7 +351,7 @@ export class Human {
// check if input changed sufficiently to trigger new detections // check if input changed sufficiently to trigger new detections
/** @hidden */ /** @hidden */
#skipFrame = async (input) => { #skipFrame = async (input) => {
if (this.config.cacheSensitivity === 0) return true; if (this.config.cacheSensitivity === 0) return false;
const resizeFact = 50; const resizeFact = 50;
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const sumT = this.tf.sum(reduced); const sumT = this.tf.sum(reduced);
@ -476,12 +484,14 @@ export class Human {
// run nanodet // run nanodet
this.analyze('Start Object:'); this.analyze('Start Object:');
if (this.config.async) { if (this.config.async) {
objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : []; if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : [];
if (this.perf.object) delete this.perf.object; if (this.perf.object) delete this.perf.object;
} else { } else {
this.state = 'run:object'; this.state = 'run:object';
timeStamp = now(); timeStamp = now();
objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : []; if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : [];
current = Math.trunc(now() - timeStamp); current = Math.trunc(now() - timeStamp);
if (current > 0) this.perf.object = current; if (current > 0) this.perf.object = current;
} }

80
src/object/centernet.ts Normal file
View File

@ -0,0 +1,80 @@
import { log, join } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import { labels } from './labels';
let model;
let last: Array<{}> = [];
let skipped = Number.MAX_SAFE_INTEGER;
export async function load(config) {
if (!model) {
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath));
const inputs = Object.values(model.modelSignature['inputs']);
model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath);
else if (config.debug) log('load model:', model.modelUrl);
} else if (config.debug) log('cached model:', model.modelUrl);
return model;
}
async function process(res, inputSize, outputShape, config) {
const results: Array<{ score: number, class: number, label: string, box: number[], boxRaw: number[] }> = [];
const detections = res.arraySync();
const squeezeT = tf.squeeze(res);
res.dispose();
const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class
squeezeT.dispose();
const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // tf.nms expects y, x
const boxesT = stackT.squeeze();
const scoresT = arr[4].squeeze();
const classesT = arr[5].squeeze();
arr.forEach((t) => t.dispose());
// @ts-ignore boxesT type is not correctly inferred
const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
boxesT.dispose();
scoresT.dispose();
classesT.dispose();
const nms = nmsT.dataSync();
nmsT.dispose();
for (const id of nms) {
const score = detections[0][id][4];
const classVal = detections[0][id][5];
const label = labels[classVal].label;
const boxRaw = [
detections[0][id][0] / inputSize,
detections[0][id][1] / inputSize,
detections[0][id][2] / inputSize,
detections[0][id][3] / inputSize,
];
const box = [
Math.trunc(boxRaw[0] * outputShape[0]),
Math.trunc(boxRaw[1] * outputShape[1]),
Math.trunc(boxRaw[2] * outputShape[0]),
Math.trunc(boxRaw[3] * outputShape[1]),
];
results.push({ score, class: classVal, label, box, boxRaw });
}
return results;
}
export async function predict(image, config) {
if (!model) return null;
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
skipped++;
return last;
}
skipped = 0;
return new Promise(async (resolve) => {
const outputSize = [image.shape[2], image.shape[1]];
const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false);
let objectT;
if (config.object.enabled) objectT = model.execute(resize, 'tower_0/detections');
resize.dispose();
const obj = await process(objectT, model.inputSize, outputSize, config);
last = obj;
resolve(obj);
});
}

0
src/nanodet/labels.ts → src/object/labels.ts
View File

2
src/nanodet/nanodet.ts → src/object/nanodet.ts
View File

@ -78,7 +78,7 @@ async function process(res, inputSize, outputShape, config) {
// normally nms is run on raw results, but since boxes need to be calculated this way we skip calulcation of // normally nms is run on raw results, but since boxes need to be calculated this way we skip calulcation of
// unnecessary boxes and run nms only on good candidates (basically it just does IOU analysis as scores are already filtered) // unnecessary boxes and run nms only on good candidates (basically it just does IOU analysis as scores are already filtered)
const nmsBoxes = results.map((a) => a.boxRaw); const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]); // switches coordinates from x,y to y,x as expected by tf.nms
const nmsScores = results.map((a) => a.score); const nmsScores = results.map((a) => a.score);
let nmsIdx: any[] = []; let nmsIdx: any[] = [];
if (nmsBoxes && nmsBoxes.length > 0) { if (nmsBoxes && nmsBoxes.length > 0) {

2
wiki

@ -1 +1 @@
Subproject commit 534d4d77d99b0fc71913e8ef6242e4c6461614f5 Subproject commit fa896c5330432f26839d362b81ea9128db60d86b