mirror of https://github.com/vladmandic/human
add experimental mb3-centernet object detection
Vladimir Mandic
parent
271b821ab7
commit
fa3ab21215
|
|
@ -9,10 +9,11 @@ import webRTC from './helpers/webrtc.js';
|
|||
let human;
|
||||
|
||||
const userConfig = {
|
||||
warmup: 'none',
|
||||
warmup: 'full',
|
||||
/*
|
||||
backend: 'webgl',
|
||||
async: true,
|
||||
async: false,
|
||||
cacheSensitivity: 0,
|
||||
filter: {
|
||||
enabled: false,
|
||||
flip: false,
|
||||
|
|
@ -26,9 +27,9 @@ const userConfig = {
|
|||
},
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: true, modelPath: 'posenet.json' },
|
||||
body: { enabled: false, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
// object: { enabled: true },
|
||||
object: { enabled: false },
|
||||
*/
|
||||
};
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -127,7 +127,7 @@ var config = {
|
|||
debug: true,
|
||||
async: true,
|
||||
warmup: "full",
|
||||
cacheSensitivity: 4e-3,
|
||||
cacheSensitivity: 5e-3,
|
||||
filter: {
|
||||
enabled: true,
|
||||
width: 0,
|
||||
|
|
@ -192,7 +192,7 @@ var config = {
|
|||
hand: {
|
||||
enabled: true,
|
||||
rotation: false,
|
||||
skipFrames: 12,
|
||||
skipFrames: 32,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
|
|
@ -206,7 +206,7 @@ var config = {
|
|||
},
|
||||
object: {
|
||||
enabled: false,
|
||||
modelPath: "nanodet.json",
|
||||
modelPath: "mb3-centernet.json",
|
||||
minConfidence: 0.2,
|
||||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
|
|
@ -236,7 +236,7 @@ function info() {
|
|||
}
|
||||
|
||||
// src/human.ts
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
var tf17 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/tfjs/backend.ts
|
||||
var tf = __toModule(require_tfjs_esm());
|
||||
|
|
@ -338,16 +338,16 @@ function getBoxCenter(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function enlargeBox(box4, factor = 1.5) {
|
||||
const center = getBoxCenter(box4);
|
||||
|
|
@ -481,11 +481,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
|
|||
return tf3.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
}
|
||||
var BlazeFaceModel = class {
|
||||
constructor(model6, config3) {
|
||||
this.model = model6;
|
||||
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]);
|
||||
constructor(model7, config3) {
|
||||
this.model = model7;
|
||||
this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
|
||||
this.anchors = tf3.tensor2d(this.anchorsData);
|
||||
this.inputSize = model6.inputs[0].shape[2];
|
||||
this.inputSize = model7.inputs[0].shape[2];
|
||||
this.config = config3;
|
||||
}
|
||||
async getBoundingBoxes(inputImage) {
|
||||
|
|
@ -534,12 +534,12 @@ var BlazeFaceModel = class {
|
|||
}
|
||||
};
|
||||
async function load(config3) {
|
||||
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")});
|
||||
const blazeFace = new BlazeFaceModel(model6, config3);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
|
||||
const blazeFace = new BlazeFaceModel(model7, config3);
|
||||
if (!model7 || !model7.modelUrl)
|
||||
log("load model failed:", config3.face.detector.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
log("load model:", model7.modelUrl);
|
||||
return blazeFace;
|
||||
}
|
||||
|
||||
|
|
@ -4174,7 +4174,7 @@ async function load3(config3) {
|
|||
log("cached model:", model.modelUrl);
|
||||
return model;
|
||||
}
|
||||
async function predict2(image12, config3, idx, count2) {
|
||||
async function predict2(image13, config3, idx, count2) {
|
||||
if (!model)
|
||||
return null;
|
||||
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
|
||||
|
|
@ -4183,7 +4183,7 @@ async function predict2(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const [red, green, blue] = tf6.split(resize, 3, 3);
|
||||
resize.dispose();
|
||||
const redNorm = tf6.mul(red, rgb[0]);
|
||||
|
|
@ -4266,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
|
|||
return best;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image12 = tf7.tidy(() => {
|
||||
const image13 = tf7.tidy(() => {
|
||||
const tensor = input.image || input.tensor || input;
|
||||
if (!(tensor instanceof tf7.Tensor))
|
||||
return null;
|
||||
|
|
@ -4275,9 +4275,9 @@ function enhance(input) {
|
|||
const norm = crop.mul(255);
|
||||
return norm;
|
||||
});
|
||||
return image12;
|
||||
return image13;
|
||||
}
|
||||
async function predict3(image12, config3, idx, count2) {
|
||||
async function predict3(image13, config3, idx, count2) {
|
||||
var _a, _b;
|
||||
if (!model2)
|
||||
return null;
|
||||
|
|
@ -4287,7 +4287,7 @@ async function predict3(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped2 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const enhanced = enhance(image12);
|
||||
const enhanced = enhance(image13);
|
||||
let resT;
|
||||
const obj = {
|
||||
age: 0,
|
||||
|
|
@ -4844,16 +4844,16 @@ function getBoxCenter2(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize2(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize2(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function scaleBoxCoordinates2(box4, factor) {
|
||||
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
|
||||
|
|
@ -16664,9 +16664,9 @@ var anchors = [
|
|||
|
||||
// src/handpose/handdetector.ts
|
||||
var HandDetector = class {
|
||||
constructor(model6) {
|
||||
constructor(model7) {
|
||||
var _a;
|
||||
this.model = model6;
|
||||
this.model = model7;
|
||||
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
|
||||
this.anchorsTensor = tf10.tensor2d(this.anchors);
|
||||
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
|
||||
|
|
@ -16720,9 +16720,9 @@ var HandDetector = class {
|
|||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image12, config3);
|
||||
image12.dispose();
|
||||
const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image13, config3);
|
||||
image13.dispose();
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
|
|
@ -16867,11 +16867,11 @@ var HandPipeline = class {
|
|||
coord[2]
|
||||
]);
|
||||
}
|
||||
async estimateHands(image12, config3) {
|
||||
async estimateHands(image13, config3) {
|
||||
let useFreshBox = false;
|
||||
let boxes;
|
||||
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image12, config3);
|
||||
boxes = await this.handDetector.estimateHandBounds(image13, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipFrame)
|
||||
|
|
@ -16890,8 +16890,8 @@ var HandPipeline = class {
|
|||
if (config3.hand.landmarks) {
|
||||
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
|
||||
const palmCenter = getBoxCenter2(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
|
||||
const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
|
||||
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
|
|
@ -17101,13 +17101,13 @@ async function load7(config3) {
|
|||
log("cached model:", model4.modelUrl);
|
||||
return model4;
|
||||
}
|
||||
async function predict6(image12, config3) {
|
||||
async function predict6(image13, config3) {
|
||||
if (!model4)
|
||||
return null;
|
||||
if (!config3.body.enabled)
|
||||
return null;
|
||||
const imgSize = {width: image12.shape[2], height: image12.shape[1]};
|
||||
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
|
||||
const imgSize = { width: image13.shape[2], height: image13.shape[1] };
|
||||
const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
|
||||
const normalize = tf13.div(resize, [255]);
|
||||
resize.dispose();
|
||||
const resT = await model4.predict(normalize);
|
||||
|
|
@ -17134,7 +17134,7 @@ async function predict6(image12, config3) {
|
|||
return [{ score, keypoints }];
|
||||
}
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var nanodet_exports = {};
|
||||
__export(nanodet_exports, {
|
||||
load: () => load8,
|
||||
|
|
@ -17142,7 +17142,7 @@ __export(nanodet_exports, {
|
|||
});
|
||||
var tf14 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/nanodet/labels.ts
|
||||
// src/object/labels.ts
|
||||
var labels = [
|
||||
{ class: 1, label: "person" },
|
||||
{ class: 2, label: "bicycle" },
|
||||
|
|
@ -17226,7 +17226,7 @@ var labels = [
|
|||
{ class: 80, label: "toothbrush" }
|
||||
];
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var model5;
|
||||
var last3 = [];
|
||||
var skipped3 = Number.MAX_SAFE_INTEGER;
|
||||
|
|
@ -17299,7 +17299,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
});
|
||||
}
|
||||
res.forEach((t) => tf14.dispose(t));
|
||||
const nmsBoxes = results.map((a) => a.boxRaw);
|
||||
const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
|
||||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
|
|
@ -17310,7 +17310,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
|
||||
return results;
|
||||
}
|
||||
async function predict7(image12, config3) {
|
||||
async function predict7(image13, config3) {
|
||||
if (!model5)
|
||||
return null;
|
||||
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
|
||||
|
|
@ -17319,8 +17319,8 @@ async function predict7(image12, config3) {
|
|||
}
|
||||
skipped3 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image12.shape[2], image12.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
|
||||
const norm = resize.div(255);
|
||||
const transpose = norm.transpose([0, 3, 1, 2]);
|
||||
norm.dispose();
|
||||
|
|
@ -17335,6 +17335,90 @@ async function predict7(image12, config3) {
|
|||
});
|
||||
}
|
||||
|
||||
// src/object/centernet.ts
|
||||
var centernet_exports = {};
|
||||
__export(centernet_exports, {
|
||||
load: () => load9,
|
||||
predict: () => predict8
|
||||
});
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var model6;
|
||||
var last4 = [];
|
||||
var skipped4 = Number.MAX_SAFE_INTEGER;
|
||||
async function load9(config3) {
|
||||
if (!model6) {
|
||||
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
|
||||
const inputs = Object.values(model6.modelSignature["inputs"]);
|
||||
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model6.inputSize)
|
||||
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
log("load model failed:", config3.object.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
} else if (config3.debug)
|
||||
log("cached model:", model6.modelUrl);
|
||||
return model6;
|
||||
}
|
||||
async function process3(res, inputSize, outputShape, config3) {
|
||||
const results = [];
|
||||
const detections = res.arraySync();
|
||||
const squeezeT = tf15.squeeze(res);
|
||||
res.dispose();
|
||||
const arr = tf15.split(squeezeT, 6, 1);
|
||||
squeezeT.dispose();
|
||||
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
|
||||
const boxesT = stackT.squeeze();
|
||||
const scoresT = arr[4].squeeze();
|
||||
const classesT = arr[5].squeeze();
|
||||
arr.forEach((t) => t.dispose());
|
||||
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
|
||||
boxesT.dispose();
|
||||
scoresT.dispose();
|
||||
classesT.dispose();
|
||||
const nms = nmsT.dataSync();
|
||||
nmsT.dispose();
|
||||
for (const id of nms) {
|
||||
const score = detections[0][id][4];
|
||||
const classVal = detections[0][id][5];
|
||||
const label = labels[classVal].label;
|
||||
const boxRaw = [
|
||||
detections[0][id][0] / inputSize,
|
||||
detections[0][id][1] / inputSize,
|
||||
detections[0][id][2] / inputSize,
|
||||
detections[0][id][3] / inputSize
|
||||
];
|
||||
const box4 = [
|
||||
Math.trunc(boxRaw[0] * outputShape[0]),
|
||||
Math.trunc(boxRaw[1] * outputShape[1]),
|
||||
Math.trunc(boxRaw[2] * outputShape[0]),
|
||||
Math.trunc(boxRaw[3] * outputShape[1])
|
||||
];
|
||||
results.push({ score, class: classVal, label, box: box4, boxRaw });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async function predict8(image13, config3) {
|
||||
if (!model6)
|
||||
return null;
|
||||
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
|
||||
skipped4++;
|
||||
return last4;
|
||||
}
|
||||
skipped4 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
|
||||
let objectT;
|
||||
if (config3.object.enabled)
|
||||
objectT = model6.execute(resize, "tower_0/detections");
|
||||
resize.dispose();
|
||||
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
|
||||
last4 = obj;
|
||||
resolve(obj);
|
||||
});
|
||||
}
|
||||
|
||||
// src/gesture/gesture.ts
|
||||
var body = (res) => {
|
||||
if (!res)
|
||||
|
|
@ -17444,7 +17528,7 @@ var hand = (res) => {
|
|||
};
|
||||
|
||||
// src/image/image.ts
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/image/imagefx.js
|
||||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
|
|
@ -17596,8 +17680,8 @@ function GLImageFilter(params) {
|
|||
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
};
|
||||
this.apply = function(image12) {
|
||||
_resize(image12.width, image12.height);
|
||||
this.apply = function(image13) {
|
||||
_resize(image13.width, image13.height);
|
||||
_drawCount = 0;
|
||||
if (!_sourceTexture)
|
||||
_sourceTexture = gl.createTexture();
|
||||
|
|
@ -17606,7 +17690,7 @@ function GLImageFilter(params) {
|
|||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
|
||||
if (_filterChain.length === 0) {
|
||||
_draw();
|
||||
return _canvas;
|
||||
|
|
@ -18157,16 +18241,16 @@ var maxSize = 2048;
|
|||
var inCanvas;
|
||||
var outCanvas;
|
||||
var fx;
|
||||
function process3(input, config3) {
|
||||
function process4(input, config3) {
|
||||
let tensor;
|
||||
if (!input)
|
||||
throw new Error("Human: Input is missing");
|
||||
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
throw new Error("Human: Input type is not recognized");
|
||||
}
|
||||
if (input instanceof tf15.Tensor) {
|
||||
if (input instanceof tf16.Tensor) {
|
||||
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
|
||||
tensor = tf15.clone(input);
|
||||
tensor = tf16.clone(input);
|
||||
else
|
||||
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
|
||||
} else {
|
||||
|
|
@ -18219,7 +18303,7 @@ function process3(input, config3) {
|
|||
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
|
||||
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
|
||||
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
|
||||
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null;
|
||||
fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
|
||||
}
|
||||
if (!fx)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
|
|
@ -18260,16 +18344,16 @@ function process3(input, config3) {
|
|||
let pixels;
|
||||
if (outCanvas.data) {
|
||||
const shape = [outCanvas.height, outCanvas.width, 3];
|
||||
pixels = tf15.tensor3d(outCanvas.data, shape, "int32");
|
||||
pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
|
||||
} else if (outCanvas instanceof ImageData) {
|
||||
pixels = tf15.browser.fromPixels(outCanvas);
|
||||
pixels = tf16.browser.fromPixels(outCanvas);
|
||||
} else if (config3.backend === "webgl" || config3.backend === "humangl") {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
tempCanvas.height = targetHeight;
|
||||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
pixels = tf15.browser.fromPixels(tempCanvas);
|
||||
pixels = tf16.browser.fromPixels(tempCanvas);
|
||||
} else {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
|
|
@ -18277,7 +18361,7 @@ function process3(input, config3) {
|
|||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
|
||||
pixels = tf15.browser.fromPixels(data);
|
||||
pixels = tf16.browser.fromPixels(data);
|
||||
}
|
||||
const casted = pixels.toFloat();
|
||||
tensor = casted.expandDims(0);
|
||||
|
|
@ -18311,7 +18395,7 @@ var options = {
|
|||
roundRect: 28,
|
||||
drawPoints: false,
|
||||
drawLabels: true,
|
||||
drawBoxes: false,
|
||||
drawBoxes: true,
|
||||
drawPolygons: true,
|
||||
fillPolygons: false,
|
||||
useDepth: true,
|
||||
|
|
@ -19537,7 +19621,7 @@ var Human = class {
|
|||
return null;
|
||||
if (!input)
|
||||
return "input is not defined";
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor))
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
|
||||
return "input must be a tensor";
|
||||
try {
|
||||
this.tf.getBackend();
|
||||
|
|
@ -19600,7 +19684,7 @@ var Human = class {
|
|||
});
|
||||
__privateAdd(this, _skipFrame, async (input) => {
|
||||
if (this.config.cacheSensitivity === 0)
|
||||
return true;
|
||||
return false;
|
||||
const resizeFact = 50;
|
||||
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
||||
const sumT = this.tf.sum(reduced);
|
||||
|
|
@ -19673,8 +19757,8 @@ var Human = class {
|
|||
if (!img)
|
||||
return null;
|
||||
let res;
|
||||
if (typeof tf16["node"] !== "undefined") {
|
||||
const data = tf16["node"].decodeJpeg(img);
|
||||
if (typeof tf17["node"] !== "undefined") {
|
||||
const data = tf17["node"].decodeJpeg(img);
|
||||
const expanded = data.expandDims(0);
|
||||
this.tf.dispose(data);
|
||||
res = await this.detect(expanded, this.config);
|
||||
|
|
@ -19685,7 +19769,7 @@ var Human = class {
|
|||
}
|
||||
return res;
|
||||
});
|
||||
this.tf = tf16;
|
||||
this.tf = tf17;
|
||||
this.draw = draw_exports;
|
||||
this.version = version;
|
||||
this.config = mergeDeep(config, userConfig);
|
||||
|
|
@ -19707,16 +19791,18 @@ var Human = class {
|
|||
emotion: null,
|
||||
embedding: null,
|
||||
nanodet: null,
|
||||
centernet: null,
|
||||
faceres: null
|
||||
};
|
||||
this.image = (input) => process3(input, this.config);
|
||||
this.image = (input) => process4(input, this.config);
|
||||
this.classes = {
|
||||
facemesh: facemesh_exports,
|
||||
emotion: emotion_exports,
|
||||
faceres: faceres_exports,
|
||||
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
|
||||
hand: handpose_exports,
|
||||
nanodet: nanodet_exports
|
||||
nanodet: nanodet_exports,
|
||||
centernet: centernet_exports
|
||||
};
|
||||
this.faceTriangulation = triangulation;
|
||||
this.faceUVMap = uvmap;
|
||||
|
|
@ -19762,6 +19848,7 @@ var Human = class {
|
|||
this.models.posenet,
|
||||
this.models.blazepose,
|
||||
this.models.nanodet,
|
||||
this.models.centernet,
|
||||
this.models.faceres
|
||||
] = await Promise.all([
|
||||
this.models.face || (this.config.face.enabled ? load2(this.config) : null),
|
||||
|
|
@ -19769,7 +19856,8 @@ var Human = class {
|
|||
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
|
||||
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
|
||||
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
|
||||
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
|
||||
]);
|
||||
} else {
|
||||
|
|
@ -19783,8 +19871,10 @@ var Human = class {
|
|||
this.models.posenet = await load5(this.config);
|
||||
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
|
||||
this.models.blazepose = await load7(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet)
|
||||
if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
|
||||
this.models.nanodet = await load8(this.config);
|
||||
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
|
||||
this.models.centernet = await load9(this.config);
|
||||
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
|
||||
this.models.faceres = await load4(this.config);
|
||||
}
|
||||
|
|
@ -19812,8 +19902,8 @@ var Human = class {
|
|||
await __privateGet(this, _checkBackend).call(this);
|
||||
await this.load();
|
||||
timeStamp = now();
|
||||
const process4 = process3(input, this.config);
|
||||
if (!process4 || !process4.tensor) {
|
||||
const process5 = process4(input, this.config);
|
||||
if (!process5 || !process5.tensor) {
|
||||
log("could not convert input to tensor");
|
||||
resolve({ error: "could not convert input to tensor" });
|
||||
return;
|
||||
|
|
@ -19821,7 +19911,7 @@ var Human = class {
|
|||
this.perf.image = Math.trunc(now() - timeStamp);
|
||||
this.analyze("Get Image:");
|
||||
timeStamp = now();
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
|
||||
if (!this.perf.frames)
|
||||
this.perf.frames = 0;
|
||||
if (!this.perf.cached)
|
||||
|
|
@ -19837,13 +19927,13 @@ var Human = class {
|
|||
let objectRes;
|
||||
let current;
|
||||
if (this.config.async) {
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
|
||||
if (this.perf.face)
|
||||
delete this.perf.face;
|
||||
} else {
|
||||
this.state = "run:face";
|
||||
timeStamp = now();
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.face = current;
|
||||
|
|
@ -19851,18 +19941,18 @@ var Human = class {
|
|||
this.analyze("Start Body:");
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
|
||||
if (this.perf.body)
|
||||
delete this.perf.body;
|
||||
} else {
|
||||
this.state = "run:body";
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.body = current;
|
||||
|
|
@ -19870,13 +19960,13 @@ var Human = class {
|
|||
this.analyze("End Body:");
|
||||
this.analyze("Start Hand:");
|
||||
if (this.config.async) {
|
||||
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
|
||||
if (this.perf.hand)
|
||||
delete this.perf.hand;
|
||||
} else {
|
||||
this.state = "run:hand";
|
||||
timeStamp = now();
|
||||
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.hand = current;
|
||||
|
|
@ -19884,13 +19974,19 @@ var Human = class {
|
|||
this.analyze("End Hand:");
|
||||
this.analyze("Start Object:");
|
||||
if (this.config.async) {
|
||||
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
|
||||
if (this.perf.object)
|
||||
delete this.perf.object;
|
||||
} else {
|
||||
this.state = "run:object";
|
||||
timeStamp = now();
|
||||
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.object = current;
|
||||
|
|
@ -19899,7 +19995,7 @@ var Human = class {
|
|||
if (this.config.async) {
|
||||
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
|
||||
}
|
||||
tf16.dispose(process4.tensor);
|
||||
tf17.dispose(process5.tensor);
|
||||
let gestureRes = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
|
|
@ -19918,7 +20014,7 @@ var Human = class {
|
|||
gesture: gestureRes,
|
||||
object: objectRes,
|
||||
performance: this.perf,
|
||||
canvas: process4.canvas
|
||||
canvas: process5.canvas
|
||||
};
|
||||
resolve(result);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ var config = {
|
|||
debug: true,
|
||||
async: true,
|
||||
warmup: "full",
|
||||
cacheSensitivity: 4e-3,
|
||||
cacheSensitivity: 5e-3,
|
||||
filter: {
|
||||
enabled: true,
|
||||
width: 0,
|
||||
|
|
@ -193,7 +193,7 @@ var config = {
|
|||
hand: {
|
||||
enabled: true,
|
||||
rotation: false,
|
||||
skipFrames: 12,
|
||||
skipFrames: 32,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
|
|
@ -207,7 +207,7 @@ var config = {
|
|||
},
|
||||
object: {
|
||||
enabled: false,
|
||||
modelPath: "nanodet.json",
|
||||
modelPath: "mb3-centernet.json",
|
||||
minConfidence: 0.2,
|
||||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
|
|
@ -237,7 +237,7 @@ function info() {
|
|||
}
|
||||
|
||||
// src/human.ts
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
var tf17 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/tfjs/backend.ts
|
||||
var tf = __toModule(require_tfjs_esm());
|
||||
|
|
@ -339,16 +339,16 @@ function getBoxCenter(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function enlargeBox(box4, factor = 1.5) {
|
||||
const center = getBoxCenter(box4);
|
||||
|
|
@ -482,11 +482,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
|
|||
return tf3.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
}
|
||||
var BlazeFaceModel = class {
|
||||
constructor(model6, config3) {
|
||||
this.model = model6;
|
||||
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]);
|
||||
constructor(model7, config3) {
|
||||
this.model = model7;
|
||||
this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
|
||||
this.anchors = tf3.tensor2d(this.anchorsData);
|
||||
this.inputSize = model6.inputs[0].shape[2];
|
||||
this.inputSize = model7.inputs[0].shape[2];
|
||||
this.config = config3;
|
||||
}
|
||||
async getBoundingBoxes(inputImage) {
|
||||
|
|
@ -535,12 +535,12 @@ var BlazeFaceModel = class {
|
|||
}
|
||||
};
|
||||
async function load(config3) {
|
||||
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")});
|
||||
const blazeFace = new BlazeFaceModel(model6, config3);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
|
||||
const blazeFace = new BlazeFaceModel(model7, config3);
|
||||
if (!model7 || !model7.modelUrl)
|
||||
log("load model failed:", config3.face.detector.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
log("load model:", model7.modelUrl);
|
||||
return blazeFace;
|
||||
}
|
||||
|
||||
|
|
@ -4175,7 +4175,7 @@ async function load3(config3) {
|
|||
log("cached model:", model.modelUrl);
|
||||
return model;
|
||||
}
|
||||
async function predict2(image12, config3, idx, count2) {
|
||||
async function predict2(image13, config3, idx, count2) {
|
||||
if (!model)
|
||||
return null;
|
||||
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
|
||||
|
|
@ -4184,7 +4184,7 @@ async function predict2(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const [red, green, blue] = tf6.split(resize, 3, 3);
|
||||
resize.dispose();
|
||||
const redNorm = tf6.mul(red, rgb[0]);
|
||||
|
|
@ -4267,7 +4267,7 @@ function match(embedding, db, threshold = 0) {
|
|||
return best;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image12 = tf7.tidy(() => {
|
||||
const image13 = tf7.tidy(() => {
|
||||
const tensor = input.image || input.tensor || input;
|
||||
if (!(tensor instanceof tf7.Tensor))
|
||||
return null;
|
||||
|
|
@ -4276,9 +4276,9 @@ function enhance(input) {
|
|||
const norm = crop.mul(255);
|
||||
return norm;
|
||||
});
|
||||
return image12;
|
||||
return image13;
|
||||
}
|
||||
async function predict3(image12, config3, idx, count2) {
|
||||
async function predict3(image13, config3, idx, count2) {
|
||||
var _a, _b;
|
||||
if (!model2)
|
||||
return null;
|
||||
|
|
@ -4288,7 +4288,7 @@ async function predict3(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped2 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const enhanced = enhance(image12);
|
||||
const enhanced = enhance(image13);
|
||||
let resT;
|
||||
const obj = {
|
||||
age: 0,
|
||||
|
|
@ -4845,16 +4845,16 @@ function getBoxCenter2(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize2(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize2(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function scaleBoxCoordinates2(box4, factor) {
|
||||
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
|
||||
|
|
@ -16665,9 +16665,9 @@ var anchors = [
|
|||
|
||||
// src/handpose/handdetector.ts
|
||||
var HandDetector = class {
|
||||
constructor(model6) {
|
||||
constructor(model7) {
|
||||
var _a;
|
||||
this.model = model6;
|
||||
this.model = model7;
|
||||
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
|
||||
this.anchorsTensor = tf10.tensor2d(this.anchors);
|
||||
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
|
||||
|
|
@ -16721,9 +16721,9 @@ var HandDetector = class {
|
|||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image12, config3);
|
||||
image12.dispose();
|
||||
const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image13, config3);
|
||||
image13.dispose();
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
|
|
@ -16868,11 +16868,11 @@ var HandPipeline = class {
|
|||
coord[2]
|
||||
]);
|
||||
}
|
||||
async estimateHands(image12, config3) {
|
||||
async estimateHands(image13, config3) {
|
||||
let useFreshBox = false;
|
||||
let boxes;
|
||||
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image12, config3);
|
||||
boxes = await this.handDetector.estimateHandBounds(image13, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipFrame)
|
||||
|
|
@ -16891,8 +16891,8 @@ var HandPipeline = class {
|
|||
if (config3.hand.landmarks) {
|
||||
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
|
||||
const palmCenter = getBoxCenter2(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
|
||||
const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
|
||||
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
|
|
@ -17102,13 +17102,13 @@ async function load7(config3) {
|
|||
log("cached model:", model4.modelUrl);
|
||||
return model4;
|
||||
}
|
||||
async function predict6(image12, config3) {
|
||||
async function predict6(image13, config3) {
|
||||
if (!model4)
|
||||
return null;
|
||||
if (!config3.body.enabled)
|
||||
return null;
|
||||
const imgSize = {width: image12.shape[2], height: image12.shape[1]};
|
||||
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
|
||||
const imgSize = { width: image13.shape[2], height: image13.shape[1] };
|
||||
const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
|
||||
const normalize = tf13.div(resize, [255]);
|
||||
resize.dispose();
|
||||
const resT = await model4.predict(normalize);
|
||||
|
|
@ -17135,7 +17135,7 @@ async function predict6(image12, config3) {
|
|||
return [{ score, keypoints }];
|
||||
}
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var nanodet_exports = {};
|
||||
__export(nanodet_exports, {
|
||||
load: () => load8,
|
||||
|
|
@ -17143,7 +17143,7 @@ __export(nanodet_exports, {
|
|||
});
|
||||
var tf14 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/nanodet/labels.ts
|
||||
// src/object/labels.ts
|
||||
var labels = [
|
||||
{ class: 1, label: "person" },
|
||||
{ class: 2, label: "bicycle" },
|
||||
|
|
@ -17227,7 +17227,7 @@ var labels = [
|
|||
{ class: 80, label: "toothbrush" }
|
||||
];
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var model5;
|
||||
var last3 = [];
|
||||
var skipped3 = Number.MAX_SAFE_INTEGER;
|
||||
|
|
@ -17300,7 +17300,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
});
|
||||
}
|
||||
res.forEach((t) => tf14.dispose(t));
|
||||
const nmsBoxes = results.map((a) => a.boxRaw);
|
||||
const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
|
||||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
|
|
@ -17311,7 +17311,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
|
||||
return results;
|
||||
}
|
||||
async function predict7(image12, config3) {
|
||||
async function predict7(image13, config3) {
|
||||
if (!model5)
|
||||
return null;
|
||||
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
|
||||
|
|
@ -17320,8 +17320,8 @@ async function predict7(image12, config3) {
|
|||
}
|
||||
skipped3 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image12.shape[2], image12.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
|
||||
const norm = resize.div(255);
|
||||
const transpose = norm.transpose([0, 3, 1, 2]);
|
||||
norm.dispose();
|
||||
|
|
@ -17336,6 +17336,90 @@ async function predict7(image12, config3) {
|
|||
});
|
||||
}
|
||||
|
||||
// src/object/centernet.ts
|
||||
var centernet_exports = {};
|
||||
__export(centernet_exports, {
|
||||
load: () => load9,
|
||||
predict: () => predict8
|
||||
});
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var model6;
|
||||
var last4 = [];
|
||||
var skipped4 = Number.MAX_SAFE_INTEGER;
|
||||
async function load9(config3) {
|
||||
if (!model6) {
|
||||
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
|
||||
const inputs = Object.values(model6.modelSignature["inputs"]);
|
||||
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model6.inputSize)
|
||||
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
log("load model failed:", config3.object.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
} else if (config3.debug)
|
||||
log("cached model:", model6.modelUrl);
|
||||
return model6;
|
||||
}
|
||||
async function process3(res, inputSize, outputShape, config3) {
|
||||
const results = [];
|
||||
const detections = res.arraySync();
|
||||
const squeezeT = tf15.squeeze(res);
|
||||
res.dispose();
|
||||
const arr = tf15.split(squeezeT, 6, 1);
|
||||
squeezeT.dispose();
|
||||
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
|
||||
const boxesT = stackT.squeeze();
|
||||
const scoresT = arr[4].squeeze();
|
||||
const classesT = arr[5].squeeze();
|
||||
arr.forEach((t) => t.dispose());
|
||||
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
|
||||
boxesT.dispose();
|
||||
scoresT.dispose();
|
||||
classesT.dispose();
|
||||
const nms = nmsT.dataSync();
|
||||
nmsT.dispose();
|
||||
for (const id of nms) {
|
||||
const score = detections[0][id][4];
|
||||
const classVal = detections[0][id][5];
|
||||
const label = labels[classVal].label;
|
||||
const boxRaw = [
|
||||
detections[0][id][0] / inputSize,
|
||||
detections[0][id][1] / inputSize,
|
||||
detections[0][id][2] / inputSize,
|
||||
detections[0][id][3] / inputSize
|
||||
];
|
||||
const box4 = [
|
||||
Math.trunc(boxRaw[0] * outputShape[0]),
|
||||
Math.trunc(boxRaw[1] * outputShape[1]),
|
||||
Math.trunc(boxRaw[2] * outputShape[0]),
|
||||
Math.trunc(boxRaw[3] * outputShape[1])
|
||||
];
|
||||
results.push({ score, class: classVal, label, box: box4, boxRaw });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async function predict8(image13, config3) {
|
||||
if (!model6)
|
||||
return null;
|
||||
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
|
||||
skipped4++;
|
||||
return last4;
|
||||
}
|
||||
skipped4 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
|
||||
let objectT;
|
||||
if (config3.object.enabled)
|
||||
objectT = model6.execute(resize, "tower_0/detections");
|
||||
resize.dispose();
|
||||
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
|
||||
last4 = obj;
|
||||
resolve(obj);
|
||||
});
|
||||
}
|
||||
|
||||
// src/gesture/gesture.ts
|
||||
var body = (res) => {
|
||||
if (!res)
|
||||
|
|
@ -17445,7 +17529,7 @@ var hand = (res) => {
|
|||
};
|
||||
|
||||
// src/image/image.ts
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/image/imagefx.js
|
||||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
|
|
@ -17597,8 +17681,8 @@ function GLImageFilter(params) {
|
|||
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
};
|
||||
this.apply = function(image12) {
|
||||
_resize(image12.width, image12.height);
|
||||
this.apply = function(image13) {
|
||||
_resize(image13.width, image13.height);
|
||||
_drawCount = 0;
|
||||
if (!_sourceTexture)
|
||||
_sourceTexture = gl.createTexture();
|
||||
|
|
@ -17607,7 +17691,7 @@ function GLImageFilter(params) {
|
|||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
|
||||
if (_filterChain.length === 0) {
|
||||
_draw();
|
||||
return _canvas;
|
||||
|
|
@ -18158,16 +18242,16 @@ var maxSize = 2048;
|
|||
var inCanvas;
|
||||
var outCanvas;
|
||||
var fx;
|
||||
function process3(input, config3) {
|
||||
function process4(input, config3) {
|
||||
let tensor;
|
||||
if (!input)
|
||||
throw new Error("Human: Input is missing");
|
||||
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
throw new Error("Human: Input type is not recognized");
|
||||
}
|
||||
if (input instanceof tf15.Tensor) {
|
||||
if (input instanceof tf16.Tensor) {
|
||||
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
|
||||
tensor = tf15.clone(input);
|
||||
tensor = tf16.clone(input);
|
||||
else
|
||||
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
|
||||
} else {
|
||||
|
|
@ -18220,7 +18304,7 @@ function process3(input, config3) {
|
|||
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
|
||||
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
|
||||
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
|
||||
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null;
|
||||
fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
|
||||
}
|
||||
if (!fx)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
|
|
@ -18261,16 +18345,16 @@ function process3(input, config3) {
|
|||
let pixels;
|
||||
if (outCanvas.data) {
|
||||
const shape = [outCanvas.height, outCanvas.width, 3];
|
||||
pixels = tf15.tensor3d(outCanvas.data, shape, "int32");
|
||||
pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
|
||||
} else if (outCanvas instanceof ImageData) {
|
||||
pixels = tf15.browser.fromPixels(outCanvas);
|
||||
pixels = tf16.browser.fromPixels(outCanvas);
|
||||
} else if (config3.backend === "webgl" || config3.backend === "humangl") {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
tempCanvas.height = targetHeight;
|
||||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
pixels = tf15.browser.fromPixels(tempCanvas);
|
||||
pixels = tf16.browser.fromPixels(tempCanvas);
|
||||
} else {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
|
|
@ -18278,7 +18362,7 @@ function process3(input, config3) {
|
|||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
|
||||
pixels = tf15.browser.fromPixels(data);
|
||||
pixels = tf16.browser.fromPixels(data);
|
||||
}
|
||||
const casted = pixels.toFloat();
|
||||
tensor = casted.expandDims(0);
|
||||
|
|
@ -18312,7 +18396,7 @@ var options = {
|
|||
roundRect: 28,
|
||||
drawPoints: false,
|
||||
drawLabels: true,
|
||||
drawBoxes: false,
|
||||
drawBoxes: true,
|
||||
drawPolygons: true,
|
||||
fillPolygons: false,
|
||||
useDepth: true,
|
||||
|
|
@ -19538,7 +19622,7 @@ var Human = class {
|
|||
return null;
|
||||
if (!input)
|
||||
return "input is not defined";
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor))
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
|
||||
return "input must be a tensor";
|
||||
try {
|
||||
this.tf.getBackend();
|
||||
|
|
@ -19601,7 +19685,7 @@ var Human = class {
|
|||
});
|
||||
__privateAdd(this, _skipFrame, async (input) => {
|
||||
if (this.config.cacheSensitivity === 0)
|
||||
return true;
|
||||
return false;
|
||||
const resizeFact = 50;
|
||||
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
||||
const sumT = this.tf.sum(reduced);
|
||||
|
|
@ -19674,8 +19758,8 @@ var Human = class {
|
|||
if (!img)
|
||||
return null;
|
||||
let res;
|
||||
if (typeof tf16["node"] !== "undefined") {
|
||||
const data = tf16["node"].decodeJpeg(img);
|
||||
if (typeof tf17["node"] !== "undefined") {
|
||||
const data = tf17["node"].decodeJpeg(img);
|
||||
const expanded = data.expandDims(0);
|
||||
this.tf.dispose(data);
|
||||
res = await this.detect(expanded, this.config);
|
||||
|
|
@ -19686,7 +19770,7 @@ var Human = class {
|
|||
}
|
||||
return res;
|
||||
});
|
||||
this.tf = tf16;
|
||||
this.tf = tf17;
|
||||
this.draw = draw_exports;
|
||||
this.version = version;
|
||||
this.config = mergeDeep(config, userConfig);
|
||||
|
|
@ -19708,16 +19792,18 @@ var Human = class {
|
|||
emotion: null,
|
||||
embedding: null,
|
||||
nanodet: null,
|
||||
centernet: null,
|
||||
faceres: null
|
||||
};
|
||||
this.image = (input) => process3(input, this.config);
|
||||
this.image = (input) => process4(input, this.config);
|
||||
this.classes = {
|
||||
facemesh: facemesh_exports,
|
||||
emotion: emotion_exports,
|
||||
faceres: faceres_exports,
|
||||
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
|
||||
hand: handpose_exports,
|
||||
nanodet: nanodet_exports
|
||||
nanodet: nanodet_exports,
|
||||
centernet: centernet_exports
|
||||
};
|
||||
this.faceTriangulation = triangulation;
|
||||
this.faceUVMap = uvmap;
|
||||
|
|
@ -19763,6 +19849,7 @@ var Human = class {
|
|||
this.models.posenet,
|
||||
this.models.blazepose,
|
||||
this.models.nanodet,
|
||||
this.models.centernet,
|
||||
this.models.faceres
|
||||
] = await Promise.all([
|
||||
this.models.face || (this.config.face.enabled ? load2(this.config) : null),
|
||||
|
|
@ -19770,7 +19857,8 @@ var Human = class {
|
|||
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
|
||||
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
|
||||
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
|
||||
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
|
||||
]);
|
||||
} else {
|
||||
|
|
@ -19784,8 +19872,10 @@ var Human = class {
|
|||
this.models.posenet = await load5(this.config);
|
||||
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
|
||||
this.models.blazepose = await load7(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet)
|
||||
if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
|
||||
this.models.nanodet = await load8(this.config);
|
||||
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
|
||||
this.models.centernet = await load9(this.config);
|
||||
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
|
||||
this.models.faceres = await load4(this.config);
|
||||
}
|
||||
|
|
@ -19813,8 +19903,8 @@ var Human = class {
|
|||
await __privateGet(this, _checkBackend).call(this);
|
||||
await this.load();
|
||||
timeStamp = now();
|
||||
const process4 = process3(input, this.config);
|
||||
if (!process4 || !process4.tensor) {
|
||||
const process5 = process4(input, this.config);
|
||||
if (!process5 || !process5.tensor) {
|
||||
log("could not convert input to tensor");
|
||||
resolve({ error: "could not convert input to tensor" });
|
||||
return;
|
||||
|
|
@ -19822,7 +19912,7 @@ var Human = class {
|
|||
this.perf.image = Math.trunc(now() - timeStamp);
|
||||
this.analyze("Get Image:");
|
||||
timeStamp = now();
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
|
||||
if (!this.perf.frames)
|
||||
this.perf.frames = 0;
|
||||
if (!this.perf.cached)
|
||||
|
|
@ -19838,13 +19928,13 @@ var Human = class {
|
|||
let objectRes;
|
||||
let current;
|
||||
if (this.config.async) {
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
|
||||
if (this.perf.face)
|
||||
delete this.perf.face;
|
||||
} else {
|
||||
this.state = "run:face";
|
||||
timeStamp = now();
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.face = current;
|
||||
|
|
@ -19852,18 +19942,18 @@ var Human = class {
|
|||
this.analyze("Start Body:");
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
|
||||
if (this.perf.body)
|
||||
delete this.perf.body;
|
||||
} else {
|
||||
this.state = "run:body";
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.body = current;
|
||||
|
|
@ -19871,13 +19961,13 @@ var Human = class {
|
|||
this.analyze("End Body:");
|
||||
this.analyze("Start Hand:");
|
||||
if (this.config.async) {
|
||||
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
|
||||
if (this.perf.hand)
|
||||
delete this.perf.hand;
|
||||
} else {
|
||||
this.state = "run:hand";
|
||||
timeStamp = now();
|
||||
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.hand = current;
|
||||
|
|
@ -19885,13 +19975,19 @@ var Human = class {
|
|||
this.analyze("End Hand:");
|
||||
this.analyze("Start Object:");
|
||||
if (this.config.async) {
|
||||
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
|
||||
if (this.perf.object)
|
||||
delete this.perf.object;
|
||||
} else {
|
||||
this.state = "run:object";
|
||||
timeStamp = now();
|
||||
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.object = current;
|
||||
|
|
@ -19900,7 +19996,7 @@ var Human = class {
|
|||
if (this.config.async) {
|
||||
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
|
||||
}
|
||||
tf16.dispose(process4.tensor);
|
||||
tf17.dispose(process5.tensor);
|
||||
let gestureRes = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
|
|
@ -19919,7 +20015,7 @@ var Human = class {
|
|||
gesture: gestureRes,
|
||||
object: objectRes,
|
||||
performance: this.perf,
|
||||
canvas: process4.canvas
|
||||
canvas: process5.canvas
|
||||
};
|
||||
resolve(result);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ var config = {
|
|||
debug: true,
|
||||
async: true,
|
||||
warmup: "full",
|
||||
cacheSensitivity: 4e-3,
|
||||
cacheSensitivity: 5e-3,
|
||||
filter: {
|
||||
enabled: true,
|
||||
width: 0,
|
||||
|
|
@ -192,7 +192,7 @@ var config = {
|
|||
hand: {
|
||||
enabled: true,
|
||||
rotation: false,
|
||||
skipFrames: 12,
|
||||
skipFrames: 32,
|
||||
minConfidence: 0.1,
|
||||
iouThreshold: 0.1,
|
||||
maxDetected: 2,
|
||||
|
|
@ -206,7 +206,7 @@ var config = {
|
|||
},
|
||||
object: {
|
||||
enabled: false,
|
||||
modelPath: "nanodet.json",
|
||||
modelPath: "mb3-centernet.json",
|
||||
minConfidence: 0.2,
|
||||
iouThreshold: 0.4,
|
||||
maxDetected: 10,
|
||||
|
|
@ -236,7 +236,7 @@ function info() {
|
|||
}
|
||||
|
||||
// src/human.ts
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
var tf17 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/tfjs/backend.ts
|
||||
var tf = __toModule(require_tfjs_esm());
|
||||
|
|
@ -338,16 +338,16 @@ function getBoxCenter(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf2.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf2.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function enlargeBox(box4, factor = 1.5) {
|
||||
const center = getBoxCenter(box4);
|
||||
|
|
@ -481,11 +481,11 @@ function decodeBounds(boxOutputs, anchors3, inputSize) {
|
|||
return tf3.concat2d([startNormalized, endNormalized], concatAxis);
|
||||
}
|
||||
var BlazeFaceModel = class {
|
||||
constructor(model6, config3) {
|
||||
this.model = model6;
|
||||
this.anchorsData = generateAnchors(model6.inputs[0].shape[1]);
|
||||
constructor(model7, config3) {
|
||||
this.model = model7;
|
||||
this.anchorsData = generateAnchors(model7.inputs[0].shape[1]);
|
||||
this.anchors = tf3.tensor2d(this.anchorsData);
|
||||
this.inputSize = model6.inputs[0].shape[2];
|
||||
this.inputSize = model7.inputs[0].shape[2];
|
||||
this.config = config3;
|
||||
}
|
||||
async getBoundingBoxes(inputImage) {
|
||||
|
|
@ -534,12 +534,12 @@ var BlazeFaceModel = class {
|
|||
}
|
||||
};
|
||||
async function load(config3) {
|
||||
const model6 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), {fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev")});
|
||||
const blazeFace = new BlazeFaceModel(model6, config3);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
const model7 = await tf3.loadGraphModel(join(config3.modelBasePath, config3.face.detector.modelPath), { fromTFHub: config3.face.detector.modelPath.includes("tfhub.dev") });
|
||||
const blazeFace = new BlazeFaceModel(model7, config3);
|
||||
if (!model7 || !model7.modelUrl)
|
||||
log("load model failed:", config3.face.detector.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
log("load model:", model7.modelUrl);
|
||||
return blazeFace;
|
||||
}
|
||||
|
||||
|
|
@ -4174,7 +4174,7 @@ async function load3(config3) {
|
|||
log("cached model:", model.modelUrl);
|
||||
return model;
|
||||
}
|
||||
async function predict2(image12, config3, idx, count2) {
|
||||
async function predict2(image13, config3, idx, count2) {
|
||||
if (!model)
|
||||
return null;
|
||||
if (skipped < config3.face.emotion.skipFrames && config3.skipFrame && lastCount === count2 && last[idx] && last[idx].length > 0) {
|
||||
|
|
@ -4183,7 +4183,7 @@ async function predict2(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const resize = tf6.image.resizeBilinear(image12, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const resize = tf6.image.resizeBilinear(image13, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const [red, green, blue] = tf6.split(resize, 3, 3);
|
||||
resize.dispose();
|
||||
const redNorm = tf6.mul(red, rgb[0]);
|
||||
|
|
@ -4266,7 +4266,7 @@ function match(embedding, db, threshold = 0) {
|
|||
return best;
|
||||
}
|
||||
function enhance(input) {
|
||||
const image12 = tf7.tidy(() => {
|
||||
const image13 = tf7.tidy(() => {
|
||||
const tensor = input.image || input.tensor || input;
|
||||
if (!(tensor instanceof tf7.Tensor))
|
||||
return null;
|
||||
|
|
@ -4275,9 +4275,9 @@ function enhance(input) {
|
|||
const norm = crop.mul(255);
|
||||
return norm;
|
||||
});
|
||||
return image12;
|
||||
return image13;
|
||||
}
|
||||
async function predict3(image12, config3, idx, count2) {
|
||||
async function predict3(image13, config3, idx, count2) {
|
||||
var _a, _b;
|
||||
if (!model2)
|
||||
return null;
|
||||
|
|
@ -4287,7 +4287,7 @@ async function predict3(image12, config3, idx, count2) {
|
|||
}
|
||||
skipped2 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const enhanced = enhance(image12);
|
||||
const enhanced = enhance(image13);
|
||||
let resT;
|
||||
const obj = {
|
||||
age: 0,
|
||||
|
|
@ -4844,16 +4844,16 @@ function getBoxCenter2(box4) {
|
|||
box4.startPoint[1] + (box4.endPoint[1] - box4.startPoint[1]) / 2
|
||||
];
|
||||
}
|
||||
function cutBoxFromImageAndResize2(box4, image12, cropSize) {
|
||||
const h = image12.shape[1];
|
||||
const w = image12.shape[2];
|
||||
function cutBoxFromImageAndResize2(box4, image13, cropSize) {
|
||||
const h = image13.shape[1];
|
||||
const w = image13.shape[2];
|
||||
const boxes = [[
|
||||
box4.startPoint[1] / h,
|
||||
box4.startPoint[0] / w,
|
||||
box4.endPoint[1] / h,
|
||||
box4.endPoint[0] / w
|
||||
]];
|
||||
return tf9.image.cropAndResize(image12, boxes, [0], cropSize);
|
||||
return tf9.image.cropAndResize(image13, boxes, [0], cropSize);
|
||||
}
|
||||
function scaleBoxCoordinates2(box4, factor) {
|
||||
const startPoint = [box4.startPoint[0] * factor[0], box4.startPoint[1] * factor[1]];
|
||||
|
|
@ -16664,9 +16664,9 @@ var anchors = [
|
|||
|
||||
// src/handpose/handdetector.ts
|
||||
var HandDetector = class {
|
||||
constructor(model6) {
|
||||
constructor(model7) {
|
||||
var _a;
|
||||
this.model = model6;
|
||||
this.model = model7;
|
||||
this.anchors = anchors.map((anchor) => [anchor.x, anchor.y]);
|
||||
this.anchorsTensor = tf10.tensor2d(this.anchors);
|
||||
this.inputSize = (_a = this.model) == null ? void 0 : _a.inputs[0].shape[2];
|
||||
|
|
@ -16720,9 +16720,9 @@ var HandDetector = class {
|
|||
async estimateHandBounds(input, config3) {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image12 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image12, config3);
|
||||
image12.dispose();
|
||||
const image13 = tf10.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image13, config3);
|
||||
image13.dispose();
|
||||
const hands = [];
|
||||
if (!predictions || predictions.length === 0)
|
||||
return hands;
|
||||
|
|
@ -16867,11 +16867,11 @@ var HandPipeline = class {
|
|||
coord[2]
|
||||
]);
|
||||
}
|
||||
async estimateHands(image12, config3) {
|
||||
async estimateHands(image13, config3) {
|
||||
let useFreshBox = false;
|
||||
let boxes;
|
||||
if (this.skipped === 0 || this.skipped > config3.hand.skipFrames || !config3.hand.landmarks || !config3.skipFrame) {
|
||||
boxes = await this.handDetector.estimateHandBounds(image12, config3);
|
||||
boxes = await this.handDetector.estimateHandBounds(image13, config3);
|
||||
this.skipped = 0;
|
||||
}
|
||||
if (config3.skipFrame)
|
||||
|
|
@ -16890,8 +16890,8 @@ var HandPipeline = class {
|
|||
if (config3.hand.landmarks) {
|
||||
const angle = config3.hand.rotation ? computeRotation2(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
|
||||
const palmCenter = getBoxCenter2(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image12.shape[2], palmCenter[1] / image12.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image12, angle, 0, palmCenterNormalized) : image12.clone();
|
||||
const palmCenterNormalized = [palmCenter[0] / image13.shape[2], palmCenter[1] / image13.shape[1]];
|
||||
const rotatedImage = config3.hand.rotation ? tf11.image.rotateWithOffset(image13, angle, 0, palmCenterNormalized) : image13.clone();
|
||||
const rotationMatrix = buildRotationMatrix2(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = cutBoxFromImageAndResize2(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
|
|
@ -17101,13 +17101,13 @@ async function load7(config3) {
|
|||
log("cached model:", model4.modelUrl);
|
||||
return model4;
|
||||
}
|
||||
async function predict6(image12, config3) {
|
||||
async function predict6(image13, config3) {
|
||||
if (!model4)
|
||||
return null;
|
||||
if (!config3.body.enabled)
|
||||
return null;
|
||||
const imgSize = {width: image12.shape[2], height: image12.shape[1]};
|
||||
const resize = tf13.image.resizeBilinear(image12, [model4.width, model4.height], false);
|
||||
const imgSize = { width: image13.shape[2], height: image13.shape[1] };
|
||||
const resize = tf13.image.resizeBilinear(image13, [model4.width, model4.height], false);
|
||||
const normalize = tf13.div(resize, [255]);
|
||||
resize.dispose();
|
||||
const resT = await model4.predict(normalize);
|
||||
|
|
@ -17134,7 +17134,7 @@ async function predict6(image12, config3) {
|
|||
return [{ score, keypoints }];
|
||||
}
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var nanodet_exports = {};
|
||||
__export(nanodet_exports, {
|
||||
load: () => load8,
|
||||
|
|
@ -17142,7 +17142,7 @@ __export(nanodet_exports, {
|
|||
});
|
||||
var tf14 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/nanodet/labels.ts
|
||||
// src/object/labels.ts
|
||||
var labels = [
|
||||
{ class: 1, label: "person" },
|
||||
{ class: 2, label: "bicycle" },
|
||||
|
|
@ -17226,7 +17226,7 @@ var labels = [
|
|||
{ class: 80, label: "toothbrush" }
|
||||
];
|
||||
|
||||
// src/nanodet/nanodet.ts
|
||||
// src/object/nanodet.ts
|
||||
var model5;
|
||||
var last3 = [];
|
||||
var skipped3 = Number.MAX_SAFE_INTEGER;
|
||||
|
|
@ -17299,7 +17299,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
});
|
||||
}
|
||||
res.forEach((t) => tf14.dispose(t));
|
||||
const nmsBoxes = results.map((a) => a.boxRaw);
|
||||
const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]);
|
||||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
|
|
@ -17310,7 +17310,7 @@ async function process2(res, inputSize, outputShape, config3) {
|
|||
results = results.filter((a, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
|
||||
return results;
|
||||
}
|
||||
async function predict7(image12, config3) {
|
||||
async function predict7(image13, config3) {
|
||||
if (!model5)
|
||||
return null;
|
||||
if (skipped3 < config3.object.skipFrames && config3.skipFrame && last3.length > 0) {
|
||||
|
|
@ -17319,8 +17319,8 @@ async function predict7(image12, config3) {
|
|||
}
|
||||
skipped3 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image12.shape[2], image12.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image12, [model5.inputSize, model5.inputSize], false);
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf14.image.resizeBilinear(image13, [model5.inputSize, model5.inputSize], false);
|
||||
const norm = resize.div(255);
|
||||
const transpose = norm.transpose([0, 3, 1, 2]);
|
||||
norm.dispose();
|
||||
|
|
@ -17335,6 +17335,90 @@ async function predict7(image12, config3) {
|
|||
});
|
||||
}
|
||||
|
||||
// src/object/centernet.ts
|
||||
var centernet_exports = {};
|
||||
__export(centernet_exports, {
|
||||
load: () => load9,
|
||||
predict: () => predict8
|
||||
});
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var model6;
|
||||
var last4 = [];
|
||||
var skipped4 = Number.MAX_SAFE_INTEGER;
|
||||
async function load9(config3) {
|
||||
if (!model6) {
|
||||
model6 = await tf15.loadGraphModel(join(config3.modelBasePath, config3.object.modelPath));
|
||||
const inputs = Object.values(model6.modelSignature["inputs"]);
|
||||
model6.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model6.inputSize)
|
||||
throw new Error(`Human: Cannot determine model inputSize: ${config3.object.modelPath}`);
|
||||
if (!model6 || !model6.modelUrl)
|
||||
log("load model failed:", config3.object.modelPath);
|
||||
else if (config3.debug)
|
||||
log("load model:", model6.modelUrl);
|
||||
} else if (config3.debug)
|
||||
log("cached model:", model6.modelUrl);
|
||||
return model6;
|
||||
}
|
||||
async function process3(res, inputSize, outputShape, config3) {
|
||||
const results = [];
|
||||
const detections = res.arraySync();
|
||||
const squeezeT = tf15.squeeze(res);
|
||||
res.dispose();
|
||||
const arr = tf15.split(squeezeT, 6, 1);
|
||||
squeezeT.dispose();
|
||||
const stackT = tf15.stack([arr[1], arr[0], arr[3], arr[2]], 1);
|
||||
const boxesT = stackT.squeeze();
|
||||
const scoresT = arr[4].squeeze();
|
||||
const classesT = arr[5].squeeze();
|
||||
arr.forEach((t) => t.dispose());
|
||||
const nmsT = await tf15.image.nonMaxSuppressionAsync(boxesT, scoresT, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
|
||||
boxesT.dispose();
|
||||
scoresT.dispose();
|
||||
classesT.dispose();
|
||||
const nms = nmsT.dataSync();
|
||||
nmsT.dispose();
|
||||
for (const id of nms) {
|
||||
const score = detections[0][id][4];
|
||||
const classVal = detections[0][id][5];
|
||||
const label = labels[classVal].label;
|
||||
const boxRaw = [
|
||||
detections[0][id][0] / inputSize,
|
||||
detections[0][id][1] / inputSize,
|
||||
detections[0][id][2] / inputSize,
|
||||
detections[0][id][3] / inputSize
|
||||
];
|
||||
const box4 = [
|
||||
Math.trunc(boxRaw[0] * outputShape[0]),
|
||||
Math.trunc(boxRaw[1] * outputShape[1]),
|
||||
Math.trunc(boxRaw[2] * outputShape[0]),
|
||||
Math.trunc(boxRaw[3] * outputShape[1])
|
||||
];
|
||||
results.push({ score, class: classVal, label, box: box4, boxRaw });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
async function predict8(image13, config3) {
|
||||
if (!model6)
|
||||
return null;
|
||||
if (skipped4 < config3.object.skipFrames && config3.skipFrame && last4.length > 0) {
|
||||
skipped4++;
|
||||
return last4;
|
||||
}
|
||||
skipped4 = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image13.shape[2], image13.shape[1]];
|
||||
const resize = tf15.image.resizeBilinear(image13, [model6.inputSize, model6.inputSize], false);
|
||||
let objectT;
|
||||
if (config3.object.enabled)
|
||||
objectT = model6.execute(resize, "tower_0/detections");
|
||||
resize.dispose();
|
||||
const obj = await process3(objectT, model6.inputSize, outputSize, config3);
|
||||
last4 = obj;
|
||||
resolve(obj);
|
||||
});
|
||||
}
|
||||
|
||||
// src/gesture/gesture.ts
|
||||
var body = (res) => {
|
||||
if (!res)
|
||||
|
|
@ -17444,7 +17528,7 @@ var hand = (res) => {
|
|||
};
|
||||
|
||||
// src/image/image.ts
|
||||
var tf15 = __toModule(require_tfjs_esm());
|
||||
var tf16 = __toModule(require_tfjs_esm());
|
||||
|
||||
// src/image/imagefx.js
|
||||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
|
|
@ -17596,8 +17680,8 @@ function GLImageFilter(params) {
|
|||
gl.uniform1f(_currentProgram.uniform.flipY, flipY ? -1 : 1);
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
};
|
||||
this.apply = function(image12) {
|
||||
_resize(image12.width, image12.height);
|
||||
this.apply = function(image13) {
|
||||
_resize(image13.width, image13.height);
|
||||
_drawCount = 0;
|
||||
if (!_sourceTexture)
|
||||
_sourceTexture = gl.createTexture();
|
||||
|
|
@ -17606,7 +17690,7 @@ function GLImageFilter(params) {
|
|||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image12);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image13);
|
||||
if (_filterChain.length === 0) {
|
||||
_draw();
|
||||
return _canvas;
|
||||
|
|
@ -18157,16 +18241,16 @@ var maxSize = 2048;
|
|||
var inCanvas;
|
||||
var outCanvas;
|
||||
var fx;
|
||||
function process3(input, config3) {
|
||||
function process4(input, config3) {
|
||||
let tensor;
|
||||
if (!input)
|
||||
throw new Error("Human: Input is missing");
|
||||
if (!(input instanceof tf15.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
if (!(input instanceof tf16.Tensor) && !(typeof Image !== "undefined" && input instanceof Image) && !(typeof ImageData !== "undefined" && input instanceof ImageData) && !(typeof ImageBitmap !== "undefined" && input instanceof ImageBitmap) && !(typeof HTMLImageElement !== "undefined" && input instanceof HTMLImageElement) && !(typeof HTMLMediaElement !== "undefined" && input instanceof HTMLMediaElement) && !(typeof HTMLVideoElement !== "undefined" && input instanceof HTMLVideoElement) && !(typeof HTMLCanvasElement !== "undefined" && input instanceof HTMLCanvasElement) && !(typeof OffscreenCanvas !== "undefined" && input instanceof OffscreenCanvas)) {
|
||||
throw new Error("Human: Input type is not recognized");
|
||||
}
|
||||
if (input instanceof tf15.Tensor) {
|
||||
if (input instanceof tf16.Tensor) {
|
||||
if (input.shape && input.shape.length === 4 && input.shape[0] === 1 && input.shape[3] === 3)
|
||||
tensor = tf15.clone(input);
|
||||
tensor = tf16.clone(input);
|
||||
else
|
||||
throw new Error(`Human: Input tensor shape must be [1, height, width, 3] and instead was ${input.shape}`);
|
||||
} else {
|
||||
|
|
@ -18219,7 +18303,7 @@ function process3(input, config3) {
|
|||
outCanvas.width = inCanvas == null ? void 0 : inCanvas.width;
|
||||
if ((outCanvas == null ? void 0 : outCanvas.height) !== (inCanvas == null ? void 0 : inCanvas.height))
|
||||
outCanvas.height = inCanvas == null ? void 0 : inCanvas.height;
|
||||
fx = tf15.ENV.flags.IS_BROWSER ? new GLImageFilter({canvas: outCanvas}) : null;
|
||||
fx = tf16.ENV.flags.IS_BROWSER ? new GLImageFilter({ canvas: outCanvas }) : null;
|
||||
}
|
||||
if (!fx)
|
||||
return { tensor: null, canvas: inCanvas };
|
||||
|
|
@ -18260,16 +18344,16 @@ function process3(input, config3) {
|
|||
let pixels;
|
||||
if (outCanvas.data) {
|
||||
const shape = [outCanvas.height, outCanvas.width, 3];
|
||||
pixels = tf15.tensor3d(outCanvas.data, shape, "int32");
|
||||
pixels = tf16.tensor3d(outCanvas.data, shape, "int32");
|
||||
} else if (outCanvas instanceof ImageData) {
|
||||
pixels = tf15.browser.fromPixels(outCanvas);
|
||||
pixels = tf16.browser.fromPixels(outCanvas);
|
||||
} else if (config3.backend === "webgl" || config3.backend === "humangl") {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
tempCanvas.height = targetHeight;
|
||||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
pixels = tf15.browser.fromPixels(tempCanvas);
|
||||
pixels = tf16.browser.fromPixels(tempCanvas);
|
||||
} else {
|
||||
const tempCanvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement("canvas");
|
||||
tempCanvas.width = targetWidth;
|
||||
|
|
@ -18277,7 +18361,7 @@ function process3(input, config3) {
|
|||
const tempCtx = tempCanvas.getContext("2d");
|
||||
tempCtx == null ? void 0 : tempCtx.drawImage(outCanvas, 0, 0);
|
||||
const data = tempCtx == null ? void 0 : tempCtx.getImageData(0, 0, targetWidth, targetHeight);
|
||||
pixels = tf15.browser.fromPixels(data);
|
||||
pixels = tf16.browser.fromPixels(data);
|
||||
}
|
||||
const casted = pixels.toFloat();
|
||||
tensor = casted.expandDims(0);
|
||||
|
|
@ -18311,7 +18395,7 @@ var options = {
|
|||
roundRect: 28,
|
||||
drawPoints: false,
|
||||
drawLabels: true,
|
||||
drawBoxes: false,
|
||||
drawBoxes: true,
|
||||
drawPolygons: true,
|
||||
fillPolygons: false,
|
||||
useDepth: true,
|
||||
|
|
@ -19537,7 +19621,7 @@ var Human = class {
|
|||
return null;
|
||||
if (!input)
|
||||
return "input is not defined";
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf16.Tensor))
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof tf17.Tensor))
|
||||
return "input must be a tensor";
|
||||
try {
|
||||
this.tf.getBackend();
|
||||
|
|
@ -19600,7 +19684,7 @@ var Human = class {
|
|||
});
|
||||
__privateAdd(this, _skipFrame, async (input) => {
|
||||
if (this.config.cacheSensitivity === 0)
|
||||
return true;
|
||||
return false;
|
||||
const resizeFact = 50;
|
||||
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
||||
const sumT = this.tf.sum(reduced);
|
||||
|
|
@ -19673,8 +19757,8 @@ var Human = class {
|
|||
if (!img)
|
||||
return null;
|
||||
let res;
|
||||
if (typeof tf16["node"] !== "undefined") {
|
||||
const data = tf16["node"].decodeJpeg(img);
|
||||
if (typeof tf17["node"] !== "undefined") {
|
||||
const data = tf17["node"].decodeJpeg(img);
|
||||
const expanded = data.expandDims(0);
|
||||
this.tf.dispose(data);
|
||||
res = await this.detect(expanded, this.config);
|
||||
|
|
@ -19685,7 +19769,7 @@ var Human = class {
|
|||
}
|
||||
return res;
|
||||
});
|
||||
this.tf = tf16;
|
||||
this.tf = tf17;
|
||||
this.draw = draw_exports;
|
||||
this.version = version;
|
||||
this.config = mergeDeep(config, userConfig);
|
||||
|
|
@ -19707,16 +19791,18 @@ var Human = class {
|
|||
emotion: null,
|
||||
embedding: null,
|
||||
nanodet: null,
|
||||
centernet: null,
|
||||
faceres: null
|
||||
};
|
||||
this.image = (input) => process3(input, this.config);
|
||||
this.image = (input) => process4(input, this.config);
|
||||
this.classes = {
|
||||
facemesh: facemesh_exports,
|
||||
emotion: emotion_exports,
|
||||
faceres: faceres_exports,
|
||||
body: this.config.body.modelPath.includes("posenet") ? posenet_exports : blazepose_exports,
|
||||
hand: handpose_exports,
|
||||
nanodet: nanodet_exports
|
||||
nanodet: nanodet_exports,
|
||||
centernet: centernet_exports
|
||||
};
|
||||
this.faceTriangulation = triangulation;
|
||||
this.faceUVMap = uvmap;
|
||||
|
|
@ -19762,6 +19848,7 @@ var Human = class {
|
|||
this.models.posenet,
|
||||
this.models.blazepose,
|
||||
this.models.nanodet,
|
||||
this.models.centernet,
|
||||
this.models.faceres
|
||||
] = await Promise.all([
|
||||
this.models.face || (this.config.face.enabled ? load2(this.config) : null),
|
||||
|
|
@ -19769,7 +19856,8 @@ var Human = class {
|
|||
this.models.handpose || (this.config.hand.enabled ? load6(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes("posenet") ? load5(this.config) : null),
|
||||
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes("blazepose") ? load7(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled ? load8(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes("nanodet") ? load8(this.config) : null),
|
||||
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes("centernet") ? load9(this.config) : null),
|
||||
this.models.faceres || (this.config.face.enabled && this.config.face.description.enabled ? load4(this.config) : null)
|
||||
]);
|
||||
} else {
|
||||
|
|
@ -19783,8 +19871,10 @@ var Human = class {
|
|||
this.models.posenet = await load5(this.config);
|
||||
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes("blazepose"))
|
||||
this.models.blazepose = await load7(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet)
|
||||
if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes("nanodet"))
|
||||
this.models.nanodet = await load8(this.config);
|
||||
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes("centernet"))
|
||||
this.models.centernet = await load9(this.config);
|
||||
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres)
|
||||
this.models.faceres = await load4(this.config);
|
||||
}
|
||||
|
|
@ -19812,8 +19902,8 @@ var Human = class {
|
|||
await __privateGet(this, _checkBackend).call(this);
|
||||
await this.load();
|
||||
timeStamp = now();
|
||||
const process4 = process3(input, this.config);
|
||||
if (!process4 || !process4.tensor) {
|
||||
const process5 = process4(input, this.config);
|
||||
if (!process5 || !process5.tensor) {
|
||||
log("could not convert input to tensor");
|
||||
resolve({ error: "could not convert input to tensor" });
|
||||
return;
|
||||
|
|
@ -19821,7 +19911,7 @@ var Human = class {
|
|||
this.perf.image = Math.trunc(now() - timeStamp);
|
||||
this.analyze("Get Image:");
|
||||
timeStamp = now();
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process4.tensor);
|
||||
this.config.skipFrame = await __privateGet(this, _skipFrame).call(this, process5.tensor);
|
||||
if (!this.perf.frames)
|
||||
this.perf.frames = 0;
|
||||
if (!this.perf.cached)
|
||||
|
|
@ -19837,13 +19927,13 @@ var Human = class {
|
|||
let objectRes;
|
||||
let current;
|
||||
if (this.config.async) {
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? detectFace(this, process5.tensor) : [];
|
||||
if (this.perf.face)
|
||||
delete this.perf.face;
|
||||
} else {
|
||||
this.state = "run:face";
|
||||
timeStamp = now();
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process4.tensor) : [];
|
||||
faceRes = this.config.face.enabled ? await detectFace(this, process5.tensor) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.face = current;
|
||||
|
|
@ -19851,18 +19941,18 @@ var Human = class {
|
|||
this.analyze("Start Body:");
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? predict6(process5.tensor, this.config) : [];
|
||||
if (this.perf.body)
|
||||
delete this.perf.body;
|
||||
} else {
|
||||
this.state = "run:body";
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes("posenet"))
|
||||
bodyRes = this.config.body.enabled ? await predict4(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict4(process5.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes("blazepose"))
|
||||
bodyRes = this.config.body.enabled ? await predict6(process4.tensor, this.config) : [];
|
||||
bodyRes = this.config.body.enabled ? await predict6(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.body = current;
|
||||
|
|
@ -19870,13 +19960,13 @@ var Human = class {
|
|||
this.analyze("End Body:");
|
||||
this.analyze("Start Hand:");
|
||||
if (this.config.async) {
|
||||
handRes = this.config.hand.enabled ? predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? predict5(process5.tensor, this.config) : [];
|
||||
if (this.perf.hand)
|
||||
delete this.perf.hand;
|
||||
} else {
|
||||
this.state = "run:hand";
|
||||
timeStamp = now();
|
||||
handRes = this.config.hand.enabled ? await predict5(process4.tensor, this.config) : [];
|
||||
handRes = this.config.hand.enabled ? await predict5(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.hand = current;
|
||||
|
|
@ -19884,13 +19974,19 @@ var Human = class {
|
|||
this.analyze("End Hand:");
|
||||
this.analyze("Start Object:");
|
||||
if (this.config.async) {
|
||||
objectRes = this.config.object.enabled ? predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? predict8(process5.tensor, this.config) : [];
|
||||
if (this.perf.object)
|
||||
delete this.perf.object;
|
||||
} else {
|
||||
this.state = "run:object";
|
||||
timeStamp = now();
|
||||
objectRes = this.config.object.enabled ? await predict7(process4.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes("nanodet"))
|
||||
objectRes = this.config.object.enabled ? await predict7(process5.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes("centernet"))
|
||||
objectRes = this.config.object.enabled ? await predict8(process5.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0)
|
||||
this.perf.object = current;
|
||||
|
|
@ -19899,7 +19995,7 @@ var Human = class {
|
|||
if (this.config.async) {
|
||||
[faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
|
||||
}
|
||||
tf16.dispose(process4.tensor);
|
||||
tf17.dispose(process5.tensor);
|
||||
let gestureRes = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
|
|
@ -19918,7 +20014,7 @@ var Human = class {
|
|||
gesture: gestureRes,
|
||||
object: objectRes,
|
||||
performance: this.perf,
|
||||
canvas: process4.canvas
|
||||
canvas: process5.canvas
|
||||
};
|
||||
resolve(result);
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -68,7 +68,7 @@
|
|||
"canvas": "^2.8.0",
|
||||
"chokidar": "^3.5.1",
|
||||
"dayjs": "^1.10.4",
|
||||
"esbuild": "^0.12.0",
|
||||
"esbuild": "^0.12.1",
|
||||
"eslint": "^7.26.0",
|
||||
"eslint-config-airbnb-base": "^14.2.1",
|
||||
"eslint-plugin-import": "^2.23.2",
|
||||
|
|
|
|||
|
|
@ -319,7 +319,8 @@ const config: Config = {
|
|||
|
||||
object: {
|
||||
enabled: false,
|
||||
modelPath: 'nanodet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath
|
||||
modelPath: 'mb3-centernet.json', // experimental: object detection model, can be absolute path or relative to modelBasePath
|
||||
// can be 'mb3-centernet' or 'nanodet'
|
||||
minConfidence: 0.2, // threshold for discarding a prediction
|
||||
iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
|
||||
maxDetected: 10, // maximum number of objects detected in the input
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ export const options: DrawOptions = {
|
|||
roundRect: <number>28,
|
||||
drawPoints: <Boolean>false,
|
||||
drawLabels: <Boolean>true,
|
||||
drawBoxes: <Boolean>false,
|
||||
drawBoxes: <Boolean>true,
|
||||
drawPolygons: <Boolean>true,
|
||||
fillPolygons: <Boolean>false,
|
||||
useDepth: <Boolean>true,
|
||||
|
|
|
|||
30
src/human.ts
30
src/human.ts
|
|
@ -11,7 +11,8 @@ import * as emotion from './emotion/emotion';
|
|||
import * as posenet from './posenet/posenet';
|
||||
import * as handpose from './handpose/handpose';
|
||||
import * as blazepose from './blazepose/blazepose';
|
||||
import * as nanodet from './nanodet/nanodet';
|
||||
import * as nanodet from './object/nanodet';
|
||||
import * as centernet from './object/centernet';
|
||||
import * as gesture from './gesture/gesture';
|
||||
import * as image from './image/image';
|
||||
import * as draw from './draw/draw';
|
||||
|
|
@ -93,6 +94,7 @@ export class Human {
|
|||
emotion: Model | null,
|
||||
embedding: Model | null,
|
||||
nanodet: Model | null,
|
||||
centernet: Model | null,
|
||||
faceres: Model | null,
|
||||
};
|
||||
/** Internal: Currently loaded classes */
|
||||
|
|
@ -102,6 +104,7 @@ export class Human {
|
|||
body: typeof posenet | typeof blazepose;
|
||||
hand: typeof handpose;
|
||||
nanodet: typeof nanodet;
|
||||
centernet: typeof centernet;
|
||||
faceres: typeof faceres;
|
||||
};
|
||||
/** Face triangualtion array of 468 points, used for triangle references between points */
|
||||
|
|
@ -148,6 +151,7 @@ export class Human {
|
|||
emotion: null,
|
||||
embedding: null,
|
||||
nanodet: null,
|
||||
centernet: null,
|
||||
faceres: null,
|
||||
};
|
||||
// export access to image processing
|
||||
|
|
@ -161,6 +165,7 @@ export class Human {
|
|||
body: this.config.body.modelPath.includes('posenet') ? posenet : blazepose,
|
||||
hand: handpose,
|
||||
nanodet,
|
||||
centernet,
|
||||
};
|
||||
this.faceTriangulation = facemesh.triangulation;
|
||||
this.faceUVMap = facemesh.uvmap;
|
||||
|
|
@ -231,7 +236,7 @@ export class Human {
|
|||
const timeStamp = now();
|
||||
if (userConfig) this.config = mergeDeep(this.config, userConfig);
|
||||
|
||||
if (this.#firstRun) {
|
||||
if (this.#firstRun) { // print version info on first run and check for correct backend setup
|
||||
if (this.config.debug) log(`version: ${this.version}`);
|
||||
if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`);
|
||||
if (this.config.debug) log('platform:', this.sysinfo.platform);
|
||||
|
|
@ -243,7 +248,7 @@ export class Human {
|
|||
if (this.config.debug) log('tf flags:', this.tf.ENV.flags);
|
||||
}
|
||||
}
|
||||
if (this.config.async) {
|
||||
if (this.config.async) { // load models concurrently
|
||||
[
|
||||
this.models.face,
|
||||
this.models.emotion,
|
||||
|
|
@ -251,6 +256,7 @@ export class Human {
|
|||
this.models.posenet,
|
||||
this.models.blazepose,
|
||||
this.models.nanodet,
|
||||
this.models.centernet,
|
||||
this.models.faceres,
|
||||
] = await Promise.all([
|
||||
this.models.face || (this.config.face.enabled ? facemesh.load(this.config) : null),
|
||||
|
|
@ -258,20 +264,22 @@ export class Human {
|
|||
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
|
||||
this.models.posenet || (this.config.body.enabled && this.config.body.modelPath.includes('posenet') ? posenet.load(this.config) : null),
|
||||
this.models.blazepose || (this.config.body.enabled && this.config.body.modelPath.includes('blazepose') ? blazepose.load(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled ? nanodet.load(this.config) : null),
|
||||
this.models.nanodet || (this.config.object.enabled && this.config.object.modelPath.includes('nanodet') ? nanodet.load(this.config) : null),
|
||||
this.models.centernet || (this.config.object.enabled && this.config.object.modelPath.includes('centernet') ? centernet.load(this.config) : null),
|
||||
this.models.faceres || ((this.config.face.enabled && this.config.face.description.enabled) ? faceres.load(this.config) : null),
|
||||
]);
|
||||
} else {
|
||||
} else { // load models sequentially
|
||||
if (this.config.face.enabled && !this.models.face) this.models.face = await facemesh.load(this.config);
|
||||
if (this.config.face.enabled && this.config.face.emotion.enabled && !this.models.emotion) this.models.emotion = await emotion.load(this.config);
|
||||
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
|
||||
if (this.config.body.enabled && !this.models.posenet && this.config.body.modelPath.includes('posenet')) this.models.posenet = await posenet.load(this.config);
|
||||
if (this.config.body.enabled && !this.models.blazepose && this.config.body.modelPath.includes('blazepose')) this.models.blazepose = await blazepose.load(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet) this.models.nanodet = await nanodet.load(this.config);
|
||||
if (this.config.object.enabled && !this.models.nanodet && this.config.object.modelPath.includes('nanodet')) this.models.nanodet = await nanodet.load(this.config);
|
||||
if (this.config.object.enabled && !this.models.centernet && this.config.object.modelPath.includes('centernet')) this.models.centernet = await centernet.load(this.config);
|
||||
if (this.config.face.enabled && this.config.face.description.enabled && !this.models.faceres) this.models.faceres = await faceres.load(this.config);
|
||||
}
|
||||
|
||||
if (this.#firstRun) {
|
||||
if (this.#firstRun) { // print memory stats on first run
|
||||
if (this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors');
|
||||
this.#firstRun = false;
|
||||
}
|
||||
|
|
@ -343,7 +351,7 @@ export class Human {
|
|||
// check if input changed sufficiently to trigger new detections
|
||||
/** @hidden */
|
||||
#skipFrame = async (input) => {
|
||||
if (this.config.cacheSensitivity === 0) return true;
|
||||
if (this.config.cacheSensitivity === 0) return false;
|
||||
const resizeFact = 50;
|
||||
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
||||
const sumT = this.tf.sum(reduced);
|
||||
|
|
@ -476,12 +484,14 @@ export class Human {
|
|||
// run nanodet
|
||||
this.analyze('Start Object:');
|
||||
if (this.config.async) {
|
||||
objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : [];
|
||||
if (this.perf.object) delete this.perf.object;
|
||||
} else {
|
||||
this.state = 'run:object';
|
||||
timeStamp = now();
|
||||
objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : [];
|
||||
current = Math.trunc(now() - timeStamp);
|
||||
if (current > 0) this.perf.object = current;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
|
||||
let model;
|
||||
let last: Array<{}> = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
export async function load(config) {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath));
|
||||
const inputs = Object.values(model.modelSignature['inputs']);
|
||||
model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
|
||||
if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath);
|
||||
else if (config.debug) log('load model:', model.modelUrl);
|
||||
} else if (config.debug) log('cached model:', model.modelUrl);
|
||||
return model;
|
||||
}
|
||||
|
||||
async function process(res, inputSize, outputShape, config) {
|
||||
const results: Array<{ score: number, class: number, label: string, box: number[], boxRaw: number[] }> = [];
|
||||
const detections = res.arraySync();
|
||||
const squeezeT = tf.squeeze(res);
|
||||
res.dispose();
|
||||
const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class
|
||||
squeezeT.dispose();
|
||||
const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // tf.nms expects y, x
|
||||
const boxesT = stackT.squeeze();
|
||||
const scoresT = arr[4].squeeze();
|
||||
const classesT = arr[5].squeeze();
|
||||
arr.forEach((t) => t.dispose());
|
||||
// @ts-ignore boxesT type is not correctly inferred
|
||||
const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
|
||||
boxesT.dispose();
|
||||
scoresT.dispose();
|
||||
classesT.dispose();
|
||||
const nms = nmsT.dataSync();
|
||||
nmsT.dispose();
|
||||
for (const id of nms) {
|
||||
const score = detections[0][id][4];
|
||||
const classVal = detections[0][id][5];
|
||||
const label = labels[classVal].label;
|
||||
const boxRaw = [
|
||||
detections[0][id][0] / inputSize,
|
||||
detections[0][id][1] / inputSize,
|
||||
detections[0][id][2] / inputSize,
|
||||
detections[0][id][3] / inputSize,
|
||||
];
|
||||
const box = [
|
||||
Math.trunc(boxRaw[0] * outputShape[0]),
|
||||
Math.trunc(boxRaw[1] * outputShape[1]),
|
||||
Math.trunc(boxRaw[2] * outputShape[0]),
|
||||
Math.trunc(boxRaw[3] * outputShape[1]),
|
||||
];
|
||||
results.push({ score, class: classVal, label, box, boxRaw });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function predict(image, config) {
|
||||
if (!model) return null;
|
||||
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
|
||||
skipped++;
|
||||
return last;
|
||||
}
|
||||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const outputSize = [image.shape[2], image.shape[1]];
|
||||
const resize = tf.image.resizeBilinear(image, [model.inputSize, model.inputSize], false);
|
||||
|
||||
let objectT;
|
||||
if (config.object.enabled) objectT = model.execute(resize, 'tower_0/detections');
|
||||
resize.dispose();
|
||||
|
||||
const obj = await process(objectT, model.inputSize, outputSize, config);
|
||||
last = obj;
|
||||
resolve(obj);
|
||||
});
|
||||
}
|
||||
|
|
@ -78,7 +78,7 @@ async function process(res, inputSize, outputShape, config) {
|
|||
|
||||
// normally nms is run on raw results, but since boxes need to be calculated this way we skip calulcation of
|
||||
// unnecessary boxes and run nms only on good candidates (basically it just does IOU analysis as scores are already filtered)
|
||||
const nmsBoxes = results.map((a) => a.boxRaw);
|
||||
const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]); // switches coordinates from x,y to y,x as expected by tf.nms
|
||||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx: any[] = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit 534d4d77d99b0fc71913e8ef6242e4c6461614f5
|
||||
Subproject commit fa896c5330432f26839d362b81ea9128db60d86b
|
||||
Loading…
Reference in New Issue