ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

change on how face box is calculated

pull/356/head
Vladimir Mandic 2021-12-27 10:59:56 -05:00
parent 027b287f26
commit 9bc8832166
17 changed files with 153 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -9,9 +9,13 @@
## Changelog
### **HEAD -> main** 2021/12/18 mandic00@live.com
### **2.5.7** 2021/12/27 mandic00@live.com
### **origin/main** 2021/12/22 mandic00@live.com
- fix posenet
### **release: 2.5.6** 2021/12/15 mandic00@live.com

2
demo/nodejs/node-simple.js
View File

@ -3,6 +3,8 @@ const Human = require('../../dist/human.node.js').default; // this is same as `@
async function main(inputFile) {
const human = new Human(); // create instance of human using default configuration
await human.load(); // optional as models would be loaded on-demand first time they are required
await human.warmup(); // optional as model warmup is performed on-demand first time its executed
const buffer = fs.readFileSync(inputFile); // read file data into buffer
const tensor = human.tf.node.decodeImage(buffer); // decode jpg data
const result = await human.detect(tensor); // run detection; will initialize backend and on-demand load models

7
src/config.ts
View File

@ -23,12 +23,6 @@ export interface FaceDetectorConfig extends GenericConfig {
minConfidence: number,
/** minimum overlap between two detected faces before one is discarded */
iouThreshold: number,
/** factor used to expand detected face before further analysis
* - default: 1.6
* - for high-quality inputs can be reduced to increase precision
* - for video inputs or low-quality inputs can be increased to allow for more flexible tracking
*/
cropFactor: number,
/** should child models perform on masked image of a face */
mask: boolean,
/** should face detection return face tensor to be used in some other extenrnal model? */
@ -330,7 +324,6 @@ const config: Config = {
skipTime: 2500,
minConfidence: 0.2,
iouThreshold: 0.1,
cropFactor: 1.6,
mask: false,
return: false,
},

35
src/face/angles.ts
View File

@ -1,6 +1,3 @@
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
const calculateGaze = (face): { bearing: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
if (!face.annotations['rightEyeIris'] || !face.annotations['leftEyeIris']) return { bearing: 0, strength: 0 };
@ -16,7 +13,6 @@ const calculateGaze = (face): { bearing: number, strength: number } => {
const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions
? [face.mesh[133][0] - face.mesh[33][0], face.mesh[23][1] - face.mesh[27][1]]
: [face.mesh[263][0] - face.mesh[362][0], face.mesh[253][1] - face.mesh[257][1]];
const eyeDiff = [ // x distance between extreme point and center point normalized with eye size
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1],
@ -24,7 +20,6 @@ const calculateGaze = (face): { bearing: number, strength: number } => {
let strength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
strength = Math.min(strength, face.boxRaw[2] / 2, face.boxRaw[3] / 2); // limit strength to half of box size to avoid clipping due to low precision
const bearing = (radians([0, 0], eyeDiff) + (Math.PI / 2)) % Math.PI; // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments and rotate clockwise 90degrees
return { bearing, strength };
};
@ -56,7 +51,7 @@ export const calculateFaceAngle = (face, imageSize): {
// 3x3 rotation matrix to Euler angles based on https://www.geometrictools.com/Documentation/EulerAngles.pdf
const rotationMatrixToEulerAngle = (r) => {
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const [r00, r01, r02, r10, r11, r12, r20, r21, r22] = r;
const [r00, _r01, _r02, r10, r11, r12, r20, r21, r22] = r;
let thetaX: number;
let thetaY: number;
let thetaZ: number;
@ -80,22 +75,17 @@ export const calculateFaceAngle = (face, imageSize): {
if (isNaN(thetaZ)) thetaZ = 0;
return { pitch: 2 * -thetaX, yaw: 2 * -thetaY, roll: 2 * -thetaZ };
};
// simple Euler angle calculation based existing 3D mesh
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const meshToEulerAngle = (mesh) => {
/*
const meshToEulerAngle = (mesh) => { // simple Euler angle calculation based existing 3D mesh
const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1);
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const angle = {
// values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center
// pitch is face move up/down
pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face
// yaw is face turn left/right
yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye
// roll is face lean left/right
roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye
return { // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center
pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face // pitch is face move up/down
yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye // yaw is face turn left/right
roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye // roll is face lean left/right
};
return angle;
};
*/
// initialize gaze and mesh
const mesh = face.meshRaw;
@ -103,12 +93,7 @@ export const calculateFaceAngle = (face, imageSize): {
const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5;
// top, bottom, left, right
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
// make the xyz coordinates proportional, independent of the image/box size
pt[0] * imageSize[0] / size,
pt[1] * imageSize[1] / size,
pt[2],
]);
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [pt[0] * imageSize[0] / size, pt[1] * imageSize[1] / size, pt[2]]); // make the xyz coordinates proportional, independent of the image/box size
const y_axis = normalize(subVectors(pts[1], pts[0]));
let x_axis = normalize(subVectors(pts[3], pts[2]));

14
src/face/blazeface.ts
View File

@ -13,6 +13,7 @@ import { env } from '../util/env';
import type { Point } from '../result';
const keypointsCount = 6;
const faceBoxScaleFactor = 1.2;
let model: GraphModel | null;
let anchors: Tensor | null = null;
let inputSize = 0;
@ -54,7 +55,7 @@ function decodeBounds(boxOutputs) {
export async function getBoxes(inputImage: Tensor, config: Config) {
// sanity check on input
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return { boxes: [] };
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return [];
const t: Record<string, Tensor> = {};
t.resized = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
@ -88,16 +89,19 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
b.squeeze = tf.squeeze(b.slice);
b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
const points = await b.bbox.data();
boxes.push({
const rawBox = {
startPoint: [points[0], points[1]] as Point,
endPoint: [points[2], points[3]] as Point,
landmarks: (await b.landmarks.array()) as Point[],
confidence,
});
};
const scaledBox = util.scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize, (inputImage.shape[1] || 0) / inputSize]);
const enlargedBox = util.enlargeBox(scaledBox, faceBoxScaleFactor);
const squaredBox = util.squarifyBox(enlargedBox);
boxes.push(squaredBox);
Object.keys(b).forEach((tensor) => tf.dispose(b[tensor]));
}
}
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] };
return boxes;
}

54
src/face/facemesh.ts
View File

@ -20,37 +20,32 @@ import type { FaceResult, FaceLandmark, Point } from '../result';
import type { Config } from '../config';
type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
let boxCache: Array<DetectBox> = [];
const cache = {
boxes: [] as DetectBox[],
skipped: Number.MAX_SAFE_INTEGER,
timestamp: 0,
};
let model: GraphModel | null = null;
let inputSize = 0;
let skipped = Number.MAX_SAFE_INTEGER;
let lastTime = 0;
export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
// reset cached boxes
const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime);
const skipFrame = skipped < (config.face.detector?.skipFrames || 0);
if (!config.skipAllowed || !skipTime || !skipFrame || boxCache.length === 0) {
const possibleBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
lastTime = now();
boxCache = []; // empty cache
for (const possible of possibleBoxes.boxes) { // extract data from detector
const boxScaled = util.scaleBoxCoordinates(possible, possibleBoxes.scaleFactor);
const detectedWidth = (boxScaled.endPoint[0] - boxScaled.startPoint[0]) / (input.shape[2] || 1000);
const calcFactor = (config.face.detector?.cropFactor || 1.6) / (detectedWidth + 0.75) / 1.34; // detected face box is not the same size as calculated face box and scale also depends on detected face size
const boxEnlarged = util.enlargeBox(boxScaled, calcFactor);
const boxSquared = util.squarifyBox(boxEnlarged);
boxCache.push(boxSquared);
}
skipped = 0;
const skipTime = (config.face.detector?.skipTime || 0) > (now() - cache.timestamp);
const skipFrame = cache.skipped < (config.face.detector?.skipFrames || 0);
if (!config.skipAllowed || !skipTime || !skipFrame || cache.boxes.length === 0) {
cache.boxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
cache.timestamp = now();
cache.skipped = 0;
} else {
skipped++;
cache.skipped++;
}
const faces: Array<FaceResult> = [];
const newCache: Array<DetectBox> = [];
let id = 0;
for (let i = 0; i < boxCache.length; i++) {
let box = boxCache[i];
for (let i = 0; i < cache.boxes.length; i++) {
const box = cache.boxes[i];
let angle = 0;
let rotationMatrix;
const face: FaceResult = { // init face result
@ -74,7 +69,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
}
face.boxScore = Math.round(100 * box.confidence) / 100;
if (!config.face.mesh?.enabled) { // mesh not enabled, return resuts from detector only
face.box = util.getClampedBox(box, input);
face.box = util.clampBox(box, input);
face.boxRaw = util.getRawBox(box, input);
face.score = face.boxScore;
face.mesh = box.landmarks.map((pt) => [
@ -99,21 +94,16 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
face.mesh = util.transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize); // get processed mesh
face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]);
for (const key of Object.keys(coords.meshAnnotations)) face.annotations[key] = coords.meshAnnotations[key].map((index) => face.mesh[index]); // add annotations
const boxCalculated = util.calculateLandmarksBoundingBox(face.mesh);
const boxEnlarged = util.enlargeBox(boxCalculated, (config.face.detector?.cropFactor || 1.6));
const boxSquared = util.squarifyBox(boxEnlarged);
box = { ...boxSquared, confidence: box.confidence }; // redefine box with mesh calculated one
face.box = util.getClampedBox(box, input); // update detected box with box around the face mesh
face.boxRaw = util.getRawBox(box, input);
face.score = face.faceScore;
newCache.push(box);
tf.dispose(face.tensor);
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(config.face.detector?.rotation, box, input, inputSize); // optional rotate once more based on mesh data
const calculatedBox = { ...util.calculateFaceBox(face.mesh, box), confidence: box.confidence, landmarks: box.landmarks };
face.box = util.clampBox(calculatedBox, input);
face.boxRaw = util.getRawBox(calculatedBox, input);
newCache.push(calculatedBox);
}
}
faces.push(face);
}
boxCache = [...newCache]; // reset cache
cache.boxes = newCache; // reset cache
return faces;
}

66
src/face/facemeshutil.ts
View File

@ -15,9 +15,9 @@ export const disposeBox = (t) => tf.dispose([t.startPoint, t.endPoint]);
export const getBoxSize = (box): [number, number] => [Math.abs(box.endPoint[0] - box.startPoint[0]), Math.abs(box.endPoint[1] - box.startPoint[1])];
export const getBoxCenter = (box): [number, number] => [box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2, box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2];
export const getBoxCenter = (box): [number, number, number] => [box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2, box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2, 1];
export const getClampedBox = (box, input): Box => (box ? [
export const clampBox = (box, input): Box => (box ? [
Math.trunc(Math.max(0, box.startPoint[0])),
Math.trunc(Math.max(0, box.startPoint[1])),
Math.trunc(Math.min((input.shape[2] || 0), box.endPoint[0]) - Math.max(0, box.startPoint[0])),
@ -37,10 +37,11 @@ export const scaleBoxCoordinates = (box, factor) => {
return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence };
};
export const cutBoxFromImageAndResize = (box, image, cropSize) => {
export const cutAndResize = (box, image, cropSize) => {
const h = image.shape[1];
const w = image.shape[2];
const crop = tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize);
const cutBox = [box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w];
const crop = tf.image.cropAndResize(image, [cutBox], [0], cropSize);
const norm = tf.div(crop, constants.tf255);
tf.dispose(crop);
return norm;
@ -61,9 +62,9 @@ export const squarifyBox = (box) => {
};
export const calculateLandmarksBoundingBox = (landmarks) => {
const xs = landmarks.map((d) => d[0]);
const ys = landmarks.map((d) => d[1]);
return { startPoint: [Math.min(...xs), Math.min(...ys)] as Point, endPoint: [Math.max(...xs), Math.max(...ys)] as Point, landmarks };
const x = landmarks.map((d) => d[0]);
const y = landmarks.map((d) => d[1]);
return { startPoint: [Math.min(...x), Math.min(...y)] as Point, endPoint: [Math.max(...x), Math.max(...y)] as Point, landmarks };
};
export const fixedRotationMatrix = [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
@ -141,19 +142,20 @@ export function generateAnchors(inputSize) {
export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputSize) {
const boxSize = getBoxSize(box);
const coordsScaled = coordsRaw.map((coord) => ([ // scaled around zero-point
boxSize[0] / inputSize * (coord[0] - inputSize / 2),
boxSize[1] / inputSize * (coord[1] - inputSize / 2),
coord[2] || 0,
(boxSize[0] / inputSize) * (coord[0] - (inputSize / 2)),
(boxSize[1] / inputSize) * (coord[1] - (inputSize / 2)),
(coord[2] || 0),
]));
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
const coordsRotationMatrix = largeAngle ? buildRotationMatrix(angle, [0, 0]) : fixedRotationMatrix;
const coordsRotated = largeAngle ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled;
const inverseRotationMatrix = largeAngle ? invertTransformMatrix(rotationMatrix) : fixedRotationMatrix;
const boxCenter = [...getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1];
const boxCenter = getBoxCenter(box);
const offsets = [dot(boxCenter, inverseRotationMatrix[0]), dot(boxCenter, inverseRotationMatrix[1])];
return coordsRotated.map((coord) => ([
Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
Math.round(coord[2] || 0),
Math.trunc(coord[0] + offsets[0]),
Math.trunc(coord[1] + offsets[1]),
Math.trunc(coord[2] || 0),
]));
}
@ -165,21 +167,43 @@ export function correctFaceRotation(rotate, box, input, inputSize) {
let rotationMatrix = fixedRotationMatrix; // default
let face; // default
if (rotate && env.kernels.includes('rotatewithoffset')) {
if (rotate && env.kernels.includes('rotatewithoffset')) { // rotateWithOffset is not defined for tfjs-node
angle = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]);
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
if (largeAngle) {
const center: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
if (largeAngle) { // perform rotation only if angle is sufficiently high
const center: Point = getBoxCenter(box);
const centerRaw: Point = [center[0] / input.shape[2], center[1] / input.shape[1]];
const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw); // rotateWithOffset is not defined for tfjs-node
const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw);
rotationMatrix = buildRotationMatrix(-angle, center);
face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]);
face = cutAndResize(box, rotated, [inputSize, inputSize]);
tf.dispose(rotated);
} else {
face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]);
face = cutAndResize(box, input, [inputSize, inputSize]);
}
} else {
face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]);
face = cutAndResize(box, input, [inputSize, inputSize]);
}
return [angle, rotationMatrix, face];
}
export const findFaceCenter = (mesh) => {
const x = mesh.map((m) => m[0]);
const y = mesh.map((m) => m[1]);
// weighted center
/*
const sum = (arr: number[]) => arr.reduce((prev, curr) => prev + curr, 0);
return [sum(x) / mesh.length, sum(y) / mesh.length];
*/
// absolute center
return [Math.min(...x) + (Math.max(...x) - Math.min(...x)) / 2, Math.min(...y) + (Math.max(...y) - Math.min(...y)) / 2];
};
export const calculateFaceBox = (mesh, previousBox) => {
const center = findFaceCenter(mesh);
const boxSize = getBoxSize(previousBox);
const calculatedBox = {
startPoint: [center[0] - boxSize[0] / 2, center[1] - boxSize[1] / 2] as Point,
endPoint: [center[0] + boxSize[0] / 2, center[1] + boxSize[1] / 2] as Point,
};
return calculatedBox;
};

6
src/hand/fingergesture.ts
View File

@ -6,11 +6,11 @@
import { Finger, FingerCurl, FingerDirection, FingerGesture } from './fingerdef';
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { thumb, index, middle, ring, pinky } = Finger;
export const { thumb, index, middle, ring, pinky } = Finger;
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { none, half, full } = FingerCurl;
export const { none, half, full } = FingerCurl;
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection;
export const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection;
// describe thumbs up gesture 👍
const ThumbsUp = new FingerGesture('thumbs up');

4
src/human.ts
View File

@ -146,7 +146,7 @@ export class Human {
// reexport draw methods
this.draw = {
options: draw.options as DrawOptions,
canvas: (input: AnyCanvas | HTMLImageElement | HTMLMediaElement | HTMLVideoElement, output: AnyCanvas) => draw.canvas(input, output),
canvas: (input: AnyCanvas | HTMLImageElement | HTMLVideoElement, output: AnyCanvas) => draw.canvas(input, output),
face: (output: AnyCanvas, result: FaceResult[], options?: Partial<DrawOptions>) => draw.face(output, result, options),
body: (output: AnyCanvas, result: BodyResult[], options?: Partial<DrawOptions>) => draw.body(output, result, options),
hand: (output: AnyCanvas, result: HandResult[], options?: Partial<DrawOptions>) => draw.hand(output, result, options),
@ -342,7 +342,7 @@ export class Human {
*/
async profile(input: Input, userConfig?: Partial<Config>): Promise<Record<string, number>> {
const profile = await this.tf.profile(() => this.detect(input, userConfig));
const kernels = {};
const kernels: Record<string, number> = {};
for (const kernel of profile.kernels) { // sum kernel time values per kernel
if (kernels[kernel.name]) kernels[kernel.name] += kernel.kernelTimeMs;
else kernels[kernel.name] = kernel.kernelTimeMs;

8
src/image/image.ts
View File

@ -24,7 +24,7 @@ const last: { inputSum: number, cacheDiff: number, sumMethod: number, inputTenso
inputTensor: undefined,
};
export function canvas(width, height): AnyCanvas {
export function canvas(width: number, height: number): AnyCanvas {
let c;
if (env.browser) { // browser defines canvas object
if (env.worker) { // if runing in web worker use OffscreenCanvas
@ -260,7 +260,7 @@ const checksum = async (input: Tensor): Promise<number> => { // use tf sum or js
};
*/
export async function skip(config, input: Tensor) {
export async function skip(config: Partial<Config>, input: Tensor) {
let skipFrame = false;
if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis
@ -290,12 +290,12 @@ export async function skip(config, input: Tensor) {
const diffRelative = diffSum[0] / (input.shape[1] || 1) / (input.shape[2] || 1) / 255 / 3; // squared difference relative to input resolution and averaged per channel
tf.dispose([last.inputTensor, t.diff, t.squared, t.sum]);
last.inputTensor = tf.clone(input);
skipFrame = diffRelative <= config.cacheSensitivity;
skipFrame = diffRelative <= (config.cacheSensitivity || 0);
}
return skipFrame;
}
export async function compare(config, input1: Tensor, input2: Tensor): Promise<number> {
export async function compare(config: Partial<Config>, input1: Tensor, input2: Tensor): Promise<number> {
const t: Record<string, Tensor> = {};
if (!input1 || !input2 || input1.shape.length !== 4 || input1.shape.length !== input2.shape.length) {
if (!config.debug) log('invalid input tensor or tensor shapes do not match:', input1.shape, input2.shape);

29
src/models.ts
View File

@ -60,7 +60,7 @@ export class Models {
export function reset(instance: Human): void {
// if (instance.config.debug) log('resetting loaded models');
for (const model of Object.keys(instance.models)) instance.models[model] = null;
for (const model of Object.keys(instance.models)) instance.models[model as keyof Models] = null;
}
/** Load method preloads all instance.configured models on-demand */
@ -71,6 +71,7 @@ export async function load(instance: Human): Promise<void> {
if (!instance.models.handskeleton && instance.config.hand.landmarks && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config);
}
if (instance.config.body.enabled && !instance.models.blazepose && instance.config.body?.modelPath?.includes('blazepose')) instance.models.blazepose = blazepose.loadPose(instance.config);
// @ts-ignore optional model
if (instance.config.body.enabled && !instance.models.blazeposedetect && instance.config.body['detector'] && instance.config.body['detector']['modelPath']) instance.models.blazeposedetect = blazepose.loadDetect(instance.config);
if (instance.config.body.enabled && !instance.models.efficientpose && instance.config.body?.modelPath?.includes('efficientpose')) instance.models.efficientpose = efficientpose.load(instance.config);
if (instance.config.body.enabled && !instance.models.movenet && instance.config.body?.modelPath?.includes('movenet')) instance.models.movenet = movenet.load(instance.config);
@ -82,9 +83,13 @@ export async function load(instance: Human): Promise<void> {
if (instance.config.face.enabled && instance.config.face.emotion?.enabled && !instance.models.emotion) instance.models.emotion = emotion.load(instance.config);
if (instance.config.face.enabled && instance.config.face.iris?.enabled && !instance.models.faceiris) instance.models.faceiris = iris.load(instance.config);
if (instance.config.face.enabled && instance.config.face.mesh?.enabled && !instance.models.facemesh) instance.models.facemesh = facemesh.load(instance.config);
// @ts-ignore optional model
if (instance.config.face.enabled && instance.config.face['gear']?.enabled && !instance.models.gear) instance.models.gear = gear.load(instance.config);
// @ts-ignore optional model
if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetage) instance.models.ssrnetage = ssrnetAge.load(instance.config);
// @ts-ignore optional model
if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetgender) instance.models.ssrnetgender = ssrnetGender.load(instance.config);
// @ts-ignore optional model
if (instance.config.face.enabled && instance.config.face['mobilefacenet']?.enabled && !instance.models.mobilefacenet) instance.models.mobilefacenet = mobilefacenet.load(instance.config);
if (instance.config.hand.enabled && !instance.models.handtrack && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handtrack = handtrack.loadDetect(instance.config);
if (instance.config.hand.enabled && instance.config.hand.landmarks && !instance.models.handskeleton && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handskeleton = handtrack.loadSkeleton(instance.config);
@ -94,7 +99,7 @@ export async function load(instance: Human): Promise<void> {
// models are loaded in parallel asynchronously so lets wait until they are actually loaded
for await (const model of Object.keys(instance.models)) {
if (instance.models[model] && typeof instance.models[model] !== 'undefined') instance.models[model] = await instance.models[model];
if (instance.models[model as keyof Models] && typeof instance.models[model as keyof Models] !== 'undefined') instance.models[model as keyof Models] = await instance.models[model as keyof Models];
}
}
@ -102,20 +107,8 @@ export async function validate(instance: Human): Promise<void> {
interface Op { name: string, category: string, op: string }
const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div'];
for (const defined of Object.keys(instance.models)) {
if (instance.models[defined]) { // check if model is loaded
let models: GraphModel[] = [];
if (Array.isArray(instance.models[defined])) {
models = instance.models[defined]
.filter((model) => (model !== null))
.map((model) => ((model && model.executor) ? model : model.model));
} else {
models = [instance.models[defined]];
}
for (const model of models) {
if (!model) {
if (instance.config.debug) log('model marked as loaded but not defined:', defined);
continue;
}
const model: GraphModel | null = instance.models[defined as keyof Models] as GraphModel | null;
if (!model) continue;
const ops: string[] = [];
// @ts-ignore // executor is a private method
const executor = model?.executor;
@ -138,8 +131,6 @@ export async function validate(instance: Human): Promise<void> {
}
}
// log('model validation ops:', defined, ops);
if (missing.length > 0 && instance.config.debug) log('model validation:', defined, missing);
}
}
if (instance.config.debug && missing.length > 0) log('model validation failed:', defined, missing);
}
}

3
src/util/env.ts
View File

@ -133,11 +133,12 @@ export class Env {
this.webgl.renderer = gl.getParameter(gl.RENDERER);
}
}
// @ts-ignore navigator.gpu is only defined when webgpu is available in browser
this.webgpu.supported = this.browser && typeof navigator['gpu'] !== 'undefined';
this.webgpu.backend = this.backends.includes('webgpu');
try {
// @ts-ignore navigator.gpu is only defined when webgpu is available in browser
if (this.webgpu.supported) this.webgpu.adapter = (await navigator['gpu'].requestAdapter()).name;
// enumerate kernels
} catch {
this.webgpu.supported = false;
}

4
src/util/interpolate.ts
View File

@ -100,8 +100,8 @@ export function calc(newResult: Result, config: Config): Result {
for (const key of Object.keys(newResult.hand[i].annotations)) { // update annotations
annotations[key] = newResult.hand[i].annotations[key] && newResult.hand[i].annotations[key][0]
? newResult.hand[i].annotations[key]
.map((val, j) => val
.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor))
.map((val, j: number) => val
.map((coord: number, k: number) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor))
: null;
}
}

4
src/util/util.ts
View File

@ -1,3 +1,5 @@
import type { Config } from '../exports';
/**
* Simple helper functions used accross codebase
*/
@ -26,7 +28,7 @@ export const now = () => {
};
// helper function: checks current config validity
export function validate(defaults, config, parent = 'config', msgs: Array<{ reason: string, where: string, expected?: string }> = []) {
export function validate(defaults: Partial<Config>, config: Partial<Config>, parent = 'config', msgs: Array<{ reason: string, where: string, expected?: string }> = []) {
for (const key of Object.keys(config)) {
if (typeof config[key] === 'object') {
validate(defaults[key], config[key], key, msgs);

16
src/warmup.ts
View File

@ -30,7 +30,7 @@ async function warmupBitmap(instance: Human) {
return res;
}
async function warmupCanvas(instance: Human) {
async function warmupCanvas(instance: Human): Promise<Result | undefined> {
return new Promise((resolve) => {
let src;
// let size = 0;
@ -57,7 +57,7 @@ async function warmupCanvas(instance: Human) {
const canvas = image.canvas(img.naturalWidth, img.naturalHeight);
if (!canvas) {
log('Warmup: Canvas not found');
resolve({});
resolve(undefined);
} else {
const ctx = canvas.getContext('2d');
if (ctx) ctx.drawImage(img, 0, 0);
@ -68,18 +68,18 @@ async function warmupCanvas(instance: Human) {
}
};
if (src) img.src = src;
else resolve(null);
else resolve(undefined);
});
}
async function warmupNode(instance: Human) {
async function warmupNode(instance: Human): Promise<Result | undefined> {
const atob = (str: string) => Buffer.from(str, 'base64');
let img;
if (instance.config.warmup === 'face') img = atob(sample.face);
if (instance.config.warmup === 'body' || instance.config.warmup === 'full') img = atob(sample.body);
if (!img) return null;
else img = atob(sample.body);
let res;
if (typeof tf['node'] !== 'undefined') {
if ('node' in tf) {
// @ts-ignore tf.node may be undefined
const data = tf['node'].decodeJpeg(img);
const expanded = data.expandDims(0);
instance.tf.dispose(data);
@ -104,7 +104,7 @@ async function warmupNode(instance: Human) {
* - only used for `webgl` and `humangl` backends
* @param userConfig?: Config
*/
export async function warmup(instance: Human, userConfig?: Partial<Config>): Promise<Result> {
export async function warmup(instance: Human, userConfig?: Partial<Config>): Promise<Result | undefined> {
const t0 = now();
instance.state = 'warmup';
if (userConfig) instance.config = mergeDeep(instance.config, userConfig) as Config;

12
test/test-main.js
View File

@ -150,11 +150,11 @@ async function verifyDetails(human) {
verify(res.face.length === 1, 'details face length', res.face.length);
for (const face of res.face) {
verify(face.score > 0.9 && face.boxScore > 0.9 && face.faceScore > 0.9, 'details face score', face.score, face.boxScore, face.faceScore);
verify(face.age > 23 && face.age < 24 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
verify(face.age > 25 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
verify(face.box.length === 4 && face.boxRaw.length === 4 && face.mesh.length === 478 && face.meshRaw.length === 478 && face.embedding.length === 1024, 'details face arrays', face.box.length, face.mesh.length, face.embedding.length);
verify(face.emotion.length === 3 && face.emotion[0].score > 0.45 && face.emotion[0].emotion === 'neutral', 'details face emotion', face.emotion.length, face.emotion[0]);
verify(face.real > 0.8, 'details face anti-spoofing', face.real);
verify(face.live > 0.8, 'details face liveness', face.live);
verify(face.emotion.length === 3 && face.emotion[0].score > 0.30 && face.emotion[0].emotion === 'fear', 'details face emotion', face.emotion.length, face.emotion[0]);
verify(face.real > 0.75, 'details face anti-spoofing', face.real);
verify(face.live > 0.75, 'details face liveness', face.live);
}
verify(res.body.length === 1, 'details body length', res.body.length);
for (const body of res.body) {
@ -365,7 +365,7 @@ async function test(Human, inputConfig) {
config.body = { minConfidence: 0.0001 };
config.hand = { minConfidence: 0.0001 };
res = await testDetect(human, 'samples/in/ai-body.jpg', 'default');
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length < 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
else log('state', 'passed: sensitive result match');
// test sensitive details face
@ -373,7 +373,7 @@ async function test(Human, inputConfig) {
if (!face || face?.box?.length !== 4 || face?.mesh?.length !== 478 || face?.embedding?.length !== 1024 || face?.rotation?.matrix?.length !== 9) {
log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length);
} else log('state', 'passed: sensitive face result match');
if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.55 || face.emotion[0].emotion !== 'neutral') log('error', 'failed: sensitive face emotion result mismatch', face?.emotion);
if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.30) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion);
else log('state', 'passed: sensitive face emotion result', face?.emotion);
// test sensitive details body

2
tsconfig.json
View File

@ -31,7 +31,7 @@
"noImplicitThis": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false,
"noUnusedLocals": false,
"noUnusedLocals": true,
"noUnusedParameters": true,
"preserveConstEnums": true,
"pretty": true,