From 9bc88321669be4792cb3ff9425b29205d49f4f7e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 27 Dec 2021 10:59:56 -0500 Subject: [PATCH] change on how face box is calculated --- CHANGELOG.md | 6 +++- demo/nodejs/node-simple.js | 2 ++ src/config.ts | 7 ---- src/face/angles.ts | 35 ++++++-------------- src/face/blazeface.ts | 14 +++++--- src/face/facemesh.ts | 54 +++++++++++++----------------- src/face/facemeshutil.ts | 66 +++++++++++++++++++++++++------------ src/hand/fingergesture.ts | 6 ++-- src/human.ts | 4 +-- src/image/image.ts | 8 ++--- src/models.ts | 67 +++++++++++++++++--------------------- src/util/env.ts | 3 +- src/util/interpolate.ts | 4 +-- src/util/util.ts | 4 ++- src/warmup.ts | 16 ++++----- test/test-main.js | 12 +++---- tsconfig.json | 2 +- 17 files changed, 153 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c58b1cf6..6f28f0ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,13 @@ ## Changelog -### **HEAD -> main** 2021/12/18 mandic00@live.com +### **2.5.7** 2021/12/27 mandic00@live.com +### **origin/main** 2021/12/22 mandic00@live.com + +- fix posenet + ### **release: 2.5.6** 2021/12/15 mandic00@live.com diff --git a/demo/nodejs/node-simple.js b/demo/nodejs/node-simple.js index 9c56a96b..667e8309 100644 --- a/demo/nodejs/node-simple.js +++ b/demo/nodejs/node-simple.js @@ -3,6 +3,8 @@ const Human = require('../../dist/human.node.js').default; // this is same as `@ async function main(inputFile) { const human = new Human(); // create instance of human using default configuration + await human.load(); // optional as models would be loaded on-demand first time they are required + await human.warmup(); // optional as model warmup is performed on-demand first time its executed const buffer = fs.readFileSync(inputFile); // read file data into buffer const tensor = human.tf.node.decodeImage(buffer); // decode jpg data const result = await human.detect(tensor); // run detection; will initialize backend and on-demand load models diff --git a/src/config.ts b/src/config.ts index edc3a658..a78460af 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,12 +23,6 @@ export interface FaceDetectorConfig extends GenericConfig { minConfidence: number, /** minimum overlap between two detected faces before one is discarded */ iouThreshold: number, - /** factor used to expand detected face before further analysis - * - default: 1.6 - * - for high-quality inputs can be reduced to increase precision - * - for video inputs or low-quality inputs can be increased to allow for more flexible tracking - */ - cropFactor: number, /** should child models perform on masked image of a face */ mask: boolean, /** should face detection return face tensor to be used in some other extenrnal model? */ @@ -330,7 +324,6 @@ const config: Config = { skipTime: 2500, minConfidence: 0.2, iouThreshold: 0.1, - cropFactor: 1.6, mask: false, return: false, }, diff --git a/src/face/angles.ts b/src/face/angles.ts index 9e23f5cb..333c23a7 100644 --- a/src/face/angles.ts +++ b/src/face/angles.ts @@ -1,6 +1,3 @@ -// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const rad2deg = (theta) => Math.round((theta * 180) / Math.PI); - const calculateGaze = (face): { bearing: number, strength: number } => { const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points if (!face.annotations['rightEyeIris'] || !face.annotations['leftEyeIris']) return { bearing: 0, strength: 0 }; @@ -16,7 +13,6 @@ const calculateGaze = (face): { bearing: number, strength: number } => { const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions ? [face.mesh[133][0] - face.mesh[33][0], face.mesh[23][1] - face.mesh[27][1]] : [face.mesh[263][0] - face.mesh[362][0], face.mesh[253][1] - face.mesh[257][1]]; - const eyeDiff = [ // x distance between extreme point and center point normalized with eye size (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1], @@ -24,7 +20,6 @@ const calculateGaze = (face): { bearing: number, strength: number } => { let strength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences strength = Math.min(strength, face.boxRaw[2] / 2, face.boxRaw[3] / 2); // limit strength to half of box size to avoid clipping due to low precision const bearing = (radians([0, 0], eyeDiff) + (Math.PI / 2)) % Math.PI; // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments and rotate clockwise 90degrees - return { bearing, strength }; }; @@ -56,7 +51,7 @@ export const calculateFaceAngle = (face, imageSize): { // 3x3 rotation matrix to Euler angles based on https://www.geometrictools.com/Documentation/EulerAngles.pdf const rotationMatrixToEulerAngle = (r) => { // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars - const [r00, r01, r02, r10, r11, r12, r20, r21, r22] = r; + const [r00, _r01, _r02, r10, r11, r12, r20, r21, r22] = r; let thetaX: number; let thetaY: number; let thetaZ: number; @@ -80,22 +75,17 @@ export const calculateFaceAngle = (face, imageSize): { if (isNaN(thetaZ)) thetaZ = 0; return { pitch: 2 * -thetaX, yaw: 2 * -thetaY, roll: 2 * -thetaZ }; }; - // simple Euler angle calculation based existing 3D mesh - // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars - const meshToEulerAngle = (mesh) => { + + /* + const meshToEulerAngle = (mesh) => { // simple Euler angle calculation based existing 3D mesh const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1); - // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars - const angle = { - // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center - // pitch is face move up/down - pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face - // yaw is face turn left/right - yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye - // roll is face lean left/right - roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye + return { // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center + pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face // pitch is face move up/down + yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye // yaw is face turn left/right + roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye // roll is face lean left/right }; - return angle; }; + */ // initialize gaze and mesh const mesh = face.meshRaw; @@ -103,12 +93,7 @@ export const calculateFaceAngle = (face, imageSize): { const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5; // top, bottom, left, right - const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ - // make the xyz coordinates proportional, independent of the image/box size - pt[0] * imageSize[0] / size, - pt[1] * imageSize[1] / size, - pt[2], - ]); + const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [pt[0] * imageSize[0] / size, pt[1] * imageSize[1] / size, pt[2]]); // make the xyz coordinates proportional, independent of the image/box size const y_axis = normalize(subVectors(pts[1], pts[0])); let x_axis = normalize(subVectors(pts[3], pts[2])); diff --git a/src/face/blazeface.ts b/src/face/blazeface.ts index 46032df7..4a0470d5 100644 --- a/src/face/blazeface.ts +++ b/src/face/blazeface.ts @@ -13,6 +13,7 @@ import { env } from '../util/env'; import type { Point } from '../result'; const keypointsCount = 6; +const faceBoxScaleFactor = 1.2; let model: GraphModel | null; let anchors: Tensor | null = null; let inputSize = 0; @@ -54,7 +55,7 @@ function decodeBounds(boxOutputs) { export async function getBoxes(inputImage: Tensor, config: Config) { // sanity check on input - if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return { boxes: [] }; + if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return []; const t: Record = {}; t.resized = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]); @@ -88,16 +89,19 @@ export async function getBoxes(inputImage: Tensor, config: Config) { b.squeeze = tf.squeeze(b.slice); b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]); const points = await b.bbox.data(); - boxes.push({ + const rawBox = { startPoint: [points[0], points[1]] as Point, endPoint: [points[2], points[3]] as Point, landmarks: (await b.landmarks.array()) as Point[], confidence, - }); + }; + const scaledBox = util.scaleBoxCoordinates(rawBox, [(inputImage.shape[2] || 0) / inputSize, (inputImage.shape[1] || 0) / inputSize]); + const enlargedBox = util.enlargeBox(scaledBox, faceBoxScaleFactor); + const squaredBox = util.squarifyBox(enlargedBox); + boxes.push(squaredBox); Object.keys(b).forEach((tensor) => tf.dispose(b[tensor])); } } - Object.keys(t).forEach((tensor) => tf.dispose(t[tensor])); - return { boxes, scaleFactor: [inputImage.shape[2] / inputSize, inputImage.shape[1] / inputSize] }; + return boxes; } diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index 405b4ada..eb418dd9 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -20,37 +20,32 @@ import type { FaceResult, FaceLandmark, Point } from '../result'; import type { Config } from '../config'; type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; -let boxCache: Array = []; + +const cache = { + boxes: [] as DetectBox[], + skipped: Number.MAX_SAFE_INTEGER, + timestamp: 0, +}; + let model: GraphModel | null = null; let inputSize = 0; -let skipped = Number.MAX_SAFE_INTEGER; -let lastTime = 0; export async function predict(input: Tensor, config: Config): Promise { // reset cached boxes - const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime); - const skipFrame = skipped < (config.face.detector?.skipFrames || 0); - if (!config.skipAllowed || !skipTime || !skipFrame || boxCache.length === 0) { - const possibleBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector - lastTime = now(); - boxCache = []; // empty cache - for (const possible of possibleBoxes.boxes) { // extract data from detector - const boxScaled = util.scaleBoxCoordinates(possible, possibleBoxes.scaleFactor); - const detectedWidth = (boxScaled.endPoint[0] - boxScaled.startPoint[0]) / (input.shape[2] || 1000); - const calcFactor = (config.face.detector?.cropFactor || 1.6) / (detectedWidth + 0.75) / 1.34; // detected face box is not the same size as calculated face box and scale also depends on detected face size - const boxEnlarged = util.enlargeBox(boxScaled, calcFactor); - const boxSquared = util.squarifyBox(boxEnlarged); - boxCache.push(boxSquared); - } - skipped = 0; + const skipTime = (config.face.detector?.skipTime || 0) > (now() - cache.timestamp); + const skipFrame = cache.skipped < (config.face.detector?.skipFrames || 0); + if (!config.skipAllowed || !skipTime || !skipFrame || cache.boxes.length === 0) { + cache.boxes = await blazeface.getBoxes(input, config); // get results from blazeface detector + cache.timestamp = now(); + cache.skipped = 0; } else { - skipped++; + cache.skipped++; } const faces: Array = []; const newCache: Array = []; let id = 0; - for (let i = 0; i < boxCache.length; i++) { - let box = boxCache[i]; + for (let i = 0; i < cache.boxes.length; i++) { + const box = cache.boxes[i]; let angle = 0; let rotationMatrix; const face: FaceResult = { // init face result @@ -74,7 +69,7 @@ export async function predict(input: Tensor, config: Config): Promise [ @@ -99,21 +94,16 @@ export async function predict(input: Tensor, config: Config): Promise [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]); for (const key of Object.keys(coords.meshAnnotations)) face.annotations[key] = coords.meshAnnotations[key].map((index) => face.mesh[index]); // add annotations - const boxCalculated = util.calculateLandmarksBoundingBox(face.mesh); - const boxEnlarged = util.enlargeBox(boxCalculated, (config.face.detector?.cropFactor || 1.6)); - const boxSquared = util.squarifyBox(boxEnlarged); - box = { ...boxSquared, confidence: box.confidence }; // redefine box with mesh calculated one - face.box = util.getClampedBox(box, input); // update detected box with box around the face mesh - face.boxRaw = util.getRawBox(box, input); face.score = face.faceScore; - newCache.push(box); - tf.dispose(face.tensor); - [angle, rotationMatrix, face.tensor] = util.correctFaceRotation(config.face.detector?.rotation, box, input, inputSize); // optional rotate once more based on mesh data + const calculatedBox = { ...util.calculateFaceBox(face.mesh, box), confidence: box.confidence, landmarks: box.landmarks }; + face.box = util.clampBox(calculatedBox, input); + face.boxRaw = util.getRawBox(calculatedBox, input); + newCache.push(calculatedBox); } } faces.push(face); } - boxCache = [...newCache]; // reset cache + cache.boxes = newCache; // reset cache return faces; } diff --git a/src/face/facemeshutil.ts b/src/face/facemeshutil.ts index a97e9cb8..fe366aaf 100644 --- a/src/face/facemeshutil.ts +++ b/src/face/facemeshutil.ts @@ -15,9 +15,9 @@ export const disposeBox = (t) => tf.dispose([t.startPoint, t.endPoint]); export const getBoxSize = (box): [number, number] => [Math.abs(box.endPoint[0] - box.startPoint[0]), Math.abs(box.endPoint[1] - box.startPoint[1])]; -export const getBoxCenter = (box): [number, number] => [box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2, box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2]; +export const getBoxCenter = (box): [number, number, number] => [box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2, box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2, 1]; -export const getClampedBox = (box, input): Box => (box ? [ +export const clampBox = (box, input): Box => (box ? [ Math.trunc(Math.max(0, box.startPoint[0])), Math.trunc(Math.max(0, box.startPoint[1])), Math.trunc(Math.min((input.shape[2] || 0), box.endPoint[0]) - Math.max(0, box.startPoint[0])), @@ -37,10 +37,11 @@ export const scaleBoxCoordinates = (box, factor) => { return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence }; }; -export const cutBoxFromImageAndResize = (box, image, cropSize) => { +export const cutAndResize = (box, image, cropSize) => { const h = image.shape[1]; const w = image.shape[2]; - const crop = tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize); + const cutBox = [box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]; + const crop = tf.image.cropAndResize(image, [cutBox], [0], cropSize); const norm = tf.div(crop, constants.tf255); tf.dispose(crop); return norm; @@ -61,9 +62,9 @@ export const squarifyBox = (box) => { }; export const calculateLandmarksBoundingBox = (landmarks) => { - const xs = landmarks.map((d) => d[0]); - const ys = landmarks.map((d) => d[1]); - return { startPoint: [Math.min(...xs), Math.min(...ys)] as Point, endPoint: [Math.max(...xs), Math.max(...ys)] as Point, landmarks }; + const x = landmarks.map((d) => d[0]); + const y = landmarks.map((d) => d[1]); + return { startPoint: [Math.min(...x), Math.min(...y)] as Point, endPoint: [Math.max(...x), Math.max(...y)] as Point, landmarks }; }; export const fixedRotationMatrix = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; @@ -141,19 +142,20 @@ export function generateAnchors(inputSize) { export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputSize) { const boxSize = getBoxSize(box); const coordsScaled = coordsRaw.map((coord) => ([ // scaled around zero-point - boxSize[0] / inputSize * (coord[0] - inputSize / 2), - boxSize[1] / inputSize * (coord[1] - inputSize / 2), - coord[2] || 0, + (boxSize[0] / inputSize) * (coord[0] - (inputSize / 2)), + (boxSize[1] / inputSize) * (coord[1] - (inputSize / 2)), + (coord[2] || 0), ])); const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); const coordsRotationMatrix = largeAngle ? buildRotationMatrix(angle, [0, 0]) : fixedRotationMatrix; const coordsRotated = largeAngle ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled; const inverseRotationMatrix = largeAngle ? invertTransformMatrix(rotationMatrix) : fixedRotationMatrix; - const boxCenter = [...getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1]; + const boxCenter = getBoxCenter(box); + const offsets = [dot(boxCenter, inverseRotationMatrix[0]), dot(boxCenter, inverseRotationMatrix[1])]; return coordsRotated.map((coord) => ([ - Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])), - Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])), - Math.round(coord[2] || 0), + Math.trunc(coord[0] + offsets[0]), + Math.trunc(coord[1] + offsets[1]), + Math.trunc(coord[2] || 0), ])); } @@ -165,21 +167,43 @@ export function correctFaceRotation(rotate, box, input, inputSize) { let rotationMatrix = fixedRotationMatrix; // default let face; // default - if (rotate && env.kernels.includes('rotatewithoffset')) { + if (rotate && env.kernels.includes('rotatewithoffset')) { // rotateWithOffset is not defined for tfjs-node angle = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]); const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); - if (largeAngle) { - const center: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); + if (largeAngle) { // perform rotation only if angle is sufficiently high + const center: Point = getBoxCenter(box); const centerRaw: Point = [center[0] / input.shape[2], center[1] / input.shape[1]]; - const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw); // rotateWithOffset is not defined for tfjs-node + const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw); rotationMatrix = buildRotationMatrix(-angle, center); - face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]); + face = cutAndResize(box, rotated, [inputSize, inputSize]); tf.dispose(rotated); } else { - face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]); + face = cutAndResize(box, input, [inputSize, inputSize]); } } else { - face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]); + face = cutAndResize(box, input, [inputSize, inputSize]); } return [angle, rotationMatrix, face]; } + +export const findFaceCenter = (mesh) => { + const x = mesh.map((m) => m[0]); + const y = mesh.map((m) => m[1]); + // weighted center + /* + const sum = (arr: number[]) => arr.reduce((prev, curr) => prev + curr, 0); + return [sum(x) / mesh.length, sum(y) / mesh.length]; + */ + // absolute center + return [Math.min(...x) + (Math.max(...x) - Math.min(...x)) / 2, Math.min(...y) + (Math.max(...y) - Math.min(...y)) / 2]; +}; + +export const calculateFaceBox = (mesh, previousBox) => { + const center = findFaceCenter(mesh); + const boxSize = getBoxSize(previousBox); + const calculatedBox = { + startPoint: [center[0] - boxSize[0] / 2, center[1] - boxSize[1] / 2] as Point, + endPoint: [center[0] + boxSize[0] / 2, center[1] + boxSize[1] / 2] as Point, + }; + return calculatedBox; +}; diff --git a/src/hand/fingergesture.ts b/src/hand/fingergesture.ts index 042a7a74..126810f2 100644 --- a/src/hand/fingergesture.ts +++ b/src/hand/fingergesture.ts @@ -6,11 +6,11 @@ import { Finger, FingerCurl, FingerDirection, FingerGesture } from './fingerdef'; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const { thumb, index, middle, ring, pinky } = Finger; +export const { thumb, index, middle, ring, pinky } = Finger; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const { none, half, full } = FingerCurl; +export const { none, half, full } = FingerCurl; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection; +export const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection; // describe thumbs up gesture 👍 const ThumbsUp = new FingerGesture('thumbs up'); diff --git a/src/human.ts b/src/human.ts index ebf23c84..aee9cd9b 100644 --- a/src/human.ts +++ b/src/human.ts @@ -146,7 +146,7 @@ export class Human { // reexport draw methods this.draw = { options: draw.options as DrawOptions, - canvas: (input: AnyCanvas | HTMLImageElement | HTMLMediaElement | HTMLVideoElement, output: AnyCanvas) => draw.canvas(input, output), + canvas: (input: AnyCanvas | HTMLImageElement | HTMLVideoElement, output: AnyCanvas) => draw.canvas(input, output), face: (output: AnyCanvas, result: FaceResult[], options?: Partial) => draw.face(output, result, options), body: (output: AnyCanvas, result: BodyResult[], options?: Partial) => draw.body(output, result, options), hand: (output: AnyCanvas, result: HandResult[], options?: Partial) => draw.hand(output, result, options), @@ -342,7 +342,7 @@ export class Human { */ async profile(input: Input, userConfig?: Partial): Promise> { const profile = await this.tf.profile(() => this.detect(input, userConfig)); - const kernels = {}; + const kernels: Record = {}; for (const kernel of profile.kernels) { // sum kernel time values per kernel if (kernels[kernel.name]) kernels[kernel.name] += kernel.kernelTimeMs; else kernels[kernel.name] = kernel.kernelTimeMs; diff --git a/src/image/image.ts b/src/image/image.ts index 997f3d30..ad1473e4 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -24,7 +24,7 @@ const last: { inputSum: number, cacheDiff: number, sumMethod: number, inputTenso inputTensor: undefined, }; -export function canvas(width, height): AnyCanvas { +export function canvas(width: number, height: number): AnyCanvas { let c; if (env.browser) { // browser defines canvas object if (env.worker) { // if runing in web worker use OffscreenCanvas @@ -260,7 +260,7 @@ const checksum = async (input: Tensor): Promise => { // use tf sum or js }; */ -export async function skip(config, input: Tensor) { +export async function skip(config: Partial, input: Tensor) { let skipFrame = false; if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis @@ -290,12 +290,12 @@ export async function skip(config, input: Tensor) { const diffRelative = diffSum[0] / (input.shape[1] || 1) / (input.shape[2] || 1) / 255 / 3; // squared difference relative to input resolution and averaged per channel tf.dispose([last.inputTensor, t.diff, t.squared, t.sum]); last.inputTensor = tf.clone(input); - skipFrame = diffRelative <= config.cacheSensitivity; + skipFrame = diffRelative <= (config.cacheSensitivity || 0); } return skipFrame; } -export async function compare(config, input1: Tensor, input2: Tensor): Promise { +export async function compare(config: Partial, input1: Tensor, input2: Tensor): Promise { const t: Record = {}; if (!input1 || !input2 || input1.shape.length !== 4 || input1.shape.length !== input2.shape.length) { if (!config.debug) log('invalid input tensor or tensor shapes do not match:', input1.shape, input2.shape); diff --git a/src/models.ts b/src/models.ts index d7ddabfd..ba25760c 100644 --- a/src/models.ts +++ b/src/models.ts @@ -60,7 +60,7 @@ export class Models { export function reset(instance: Human): void { // if (instance.config.debug) log('resetting loaded models'); - for (const model of Object.keys(instance.models)) instance.models[model] = null; + for (const model of Object.keys(instance.models)) instance.models[model as keyof Models] = null; } /** Load method preloads all instance.configured models on-demand */ @@ -71,6 +71,7 @@ export async function load(instance: Human): Promise { if (!instance.models.handskeleton && instance.config.hand.landmarks && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config); } if (instance.config.body.enabled && !instance.models.blazepose && instance.config.body?.modelPath?.includes('blazepose')) instance.models.blazepose = blazepose.loadPose(instance.config); + // @ts-ignore optional model if (instance.config.body.enabled && !instance.models.blazeposedetect && instance.config.body['detector'] && instance.config.body['detector']['modelPath']) instance.models.blazeposedetect = blazepose.loadDetect(instance.config); if (instance.config.body.enabled && !instance.models.efficientpose && instance.config.body?.modelPath?.includes('efficientpose')) instance.models.efficientpose = efficientpose.load(instance.config); if (instance.config.body.enabled && !instance.models.movenet && instance.config.body?.modelPath?.includes('movenet')) instance.models.movenet = movenet.load(instance.config); @@ -82,9 +83,13 @@ export async function load(instance: Human): Promise { if (instance.config.face.enabled && instance.config.face.emotion?.enabled && !instance.models.emotion) instance.models.emotion = emotion.load(instance.config); if (instance.config.face.enabled && instance.config.face.iris?.enabled && !instance.models.faceiris) instance.models.faceiris = iris.load(instance.config); if (instance.config.face.enabled && instance.config.face.mesh?.enabled && !instance.models.facemesh) instance.models.facemesh = facemesh.load(instance.config); + // @ts-ignore optional model if (instance.config.face.enabled && instance.config.face['gear']?.enabled && !instance.models.gear) instance.models.gear = gear.load(instance.config); + // @ts-ignore optional model if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetage) instance.models.ssrnetage = ssrnetAge.load(instance.config); + // @ts-ignore optional model if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetgender) instance.models.ssrnetgender = ssrnetGender.load(instance.config); + // @ts-ignore optional model if (instance.config.face.enabled && instance.config.face['mobilefacenet']?.enabled && !instance.models.mobilefacenet) instance.models.mobilefacenet = mobilefacenet.load(instance.config); if (instance.config.hand.enabled && !instance.models.handtrack && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handtrack = handtrack.loadDetect(instance.config); if (instance.config.hand.enabled && instance.config.hand.landmarks && !instance.models.handskeleton && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handskeleton = handtrack.loadSkeleton(instance.config); @@ -94,7 +99,7 @@ export async function load(instance: Human): Promise { // models are loaded in parallel asynchronously so lets wait until they are actually loaded for await (const model of Object.keys(instance.models)) { - if (instance.models[model] && typeof instance.models[model] !== 'undefined') instance.models[model] = await instance.models[model]; + if (instance.models[model as keyof Models] && typeof instance.models[model as keyof Models] !== 'undefined') instance.models[model as keyof Models] = await instance.models[model as keyof Models]; } } @@ -102,44 +107,30 @@ export async function validate(instance: Human): Promise { interface Op { name: string, category: string, op: string } const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div']; for (const defined of Object.keys(instance.models)) { - if (instance.models[defined]) { // check if model is loaded - let models: GraphModel[] = []; - if (Array.isArray(instance.models[defined])) { - models = instance.models[defined] - .filter((model) => (model !== null)) - .map((model) => ((model && model.executor) ? model : model.model)); - } else { - models = [instance.models[defined]]; + const model: GraphModel | null = instance.models[defined as keyof Models] as GraphModel | null; + if (!model) continue; + const ops: string[] = []; + // @ts-ignore // executor is a private method + const executor = model?.executor; + if (executor && executor.graph.nodes) { + for (const kernel of Object.values(executor.graph.nodes)) { + const op = (kernel as Op).op.toLowerCase(); + if (!ops.includes(op)) ops.push(op); } - for (const model of models) { - if (!model) { - if (instance.config.debug) log('model marked as loaded but not defined:', defined); - continue; - } - const ops: string[] = []; - // @ts-ignore // executor is a private method - const executor = model?.executor; - if (executor && executor.graph.nodes) { - for (const kernel of Object.values(executor.graph.nodes)) { - const op = (kernel as Op).op.toLowerCase(); - if (!ops.includes(op)) ops.push(op); - } - } else { - if (!executor && instance.config.debug) log('model signature not determined:', defined); - } - const missing: string[] = []; - for (const op of ops) { - if (!simpleOps.includes(op) // exclude simple ops - && !instance.env.kernels.includes(op) // check actual kernel ops - && !instance.env.kernels.includes(op.replace('_', '')) // check variation without _ - && !instance.env.kernels.includes(op.replace('native', '')) // check standard variation - && !instance.env.kernels.includes(op.replace('v2', ''))) { // check non-versioned variation - missing.push(op); - } - } - // log('model validation ops:', defined, ops); - if (missing.length > 0 && instance.config.debug) log('model validation:', defined, missing); + } else { + if (!executor && instance.config.debug) log('model signature not determined:', defined); + } + const missing: string[] = []; + for (const op of ops) { + if (!simpleOps.includes(op) // exclude simple ops + && !instance.env.kernels.includes(op) // check actual kernel ops + && !instance.env.kernels.includes(op.replace('_', '')) // check variation without _ + && !instance.env.kernels.includes(op.replace('native', '')) // check standard variation + && !instance.env.kernels.includes(op.replace('v2', ''))) { // check non-versioned variation + missing.push(op); } } + // log('model validation ops:', defined, ops); + if (instance.config.debug && missing.length > 0) log('model validation failed:', defined, missing); } } diff --git a/src/util/env.ts b/src/util/env.ts index c183f879..81ecf87f 100644 --- a/src/util/env.ts +++ b/src/util/env.ts @@ -133,11 +133,12 @@ export class Env { this.webgl.renderer = gl.getParameter(gl.RENDERER); } } + // @ts-ignore navigator.gpu is only defined when webgpu is available in browser this.webgpu.supported = this.browser && typeof navigator['gpu'] !== 'undefined'; this.webgpu.backend = this.backends.includes('webgpu'); try { + // @ts-ignore navigator.gpu is only defined when webgpu is available in browser if (this.webgpu.supported) this.webgpu.adapter = (await navigator['gpu'].requestAdapter()).name; - // enumerate kernels } catch { this.webgpu.supported = false; } diff --git a/src/util/interpolate.ts b/src/util/interpolate.ts index ebae4a08..aa64ecea 100644 --- a/src/util/interpolate.ts +++ b/src/util/interpolate.ts @@ -100,8 +100,8 @@ export function calc(newResult: Result, config: Config): Result { for (const key of Object.keys(newResult.hand[i].annotations)) { // update annotations annotations[key] = newResult.hand[i].annotations[key] && newResult.hand[i].annotations[key][0] ? newResult.hand[i].annotations[key] - .map((val, j) => val - .map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor)) + .map((val, j: number) => val + .map((coord: number, k: number) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor)) : null; } } diff --git a/src/util/util.ts b/src/util/util.ts index 138e4430..2febb95e 100644 --- a/src/util/util.ts +++ b/src/util/util.ts @@ -1,3 +1,5 @@ +import type { Config } from '../exports'; + /** * Simple helper functions used accross codebase */ @@ -26,7 +28,7 @@ export const now = () => { }; // helper function: checks current config validity -export function validate(defaults, config, parent = 'config', msgs: Array<{ reason: string, where: string, expected?: string }> = []) { +export function validate(defaults: Partial, config: Partial, parent = 'config', msgs: Array<{ reason: string, where: string, expected?: string }> = []) { for (const key of Object.keys(config)) { if (typeof config[key] === 'object') { validate(defaults[key], config[key], key, msgs); diff --git a/src/warmup.ts b/src/warmup.ts index 95d97878..f0574124 100644 --- a/src/warmup.ts +++ b/src/warmup.ts @@ -30,7 +30,7 @@ async function warmupBitmap(instance: Human) { return res; } -async function warmupCanvas(instance: Human) { +async function warmupCanvas(instance: Human): Promise { return new Promise((resolve) => { let src; // let size = 0; @@ -57,7 +57,7 @@ async function warmupCanvas(instance: Human) { const canvas = image.canvas(img.naturalWidth, img.naturalHeight); if (!canvas) { log('Warmup: Canvas not found'); - resolve({}); + resolve(undefined); } else { const ctx = canvas.getContext('2d'); if (ctx) ctx.drawImage(img, 0, 0); @@ -68,18 +68,18 @@ async function warmupCanvas(instance: Human) { } }; if (src) img.src = src; - else resolve(null); + else resolve(undefined); }); } -async function warmupNode(instance: Human) { +async function warmupNode(instance: Human): Promise { const atob = (str: string) => Buffer.from(str, 'base64'); let img; if (instance.config.warmup === 'face') img = atob(sample.face); - if (instance.config.warmup === 'body' || instance.config.warmup === 'full') img = atob(sample.body); - if (!img) return null; + else img = atob(sample.body); let res; - if (typeof tf['node'] !== 'undefined') { + if ('node' in tf) { + // @ts-ignore tf.node may be undefined const data = tf['node'].decodeJpeg(img); const expanded = data.expandDims(0); instance.tf.dispose(data); @@ -104,7 +104,7 @@ async function warmupNode(instance: Human) { * - only used for `webgl` and `humangl` backends * @param userConfig?: Config */ -export async function warmup(instance: Human, userConfig?: Partial): Promise { +export async function warmup(instance: Human, userConfig?: Partial): Promise { const t0 = now(); instance.state = 'warmup'; if (userConfig) instance.config = mergeDeep(instance.config, userConfig) as Config; diff --git a/test/test-main.js b/test/test-main.js index b55edc04..39c79c67 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -150,11 +150,11 @@ async function verifyDetails(human) { verify(res.face.length === 1, 'details face length', res.face.length); for (const face of res.face) { verify(face.score > 0.9 && face.boxScore > 0.9 && face.faceScore > 0.9, 'details face score', face.score, face.boxScore, face.faceScore); - verify(face.age > 23 && face.age < 24 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris); + verify(face.age > 25 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris); verify(face.box.length === 4 && face.boxRaw.length === 4 && face.mesh.length === 478 && face.meshRaw.length === 478 && face.embedding.length === 1024, 'details face arrays', face.box.length, face.mesh.length, face.embedding.length); - verify(face.emotion.length === 3 && face.emotion[0].score > 0.45 && face.emotion[0].emotion === 'neutral', 'details face emotion', face.emotion.length, face.emotion[0]); - verify(face.real > 0.8, 'details face anti-spoofing', face.real); - verify(face.live > 0.8, 'details face liveness', face.live); + verify(face.emotion.length === 3 && face.emotion[0].score > 0.30 && face.emotion[0].emotion === 'fear', 'details face emotion', face.emotion.length, face.emotion[0]); + verify(face.real > 0.75, 'details face anti-spoofing', face.real); + verify(face.live > 0.75, 'details face liveness', face.live); } verify(res.body.length === 1, 'details body length', res.body.length); for (const body of res.body) { @@ -365,7 +365,7 @@ async function test(Human, inputConfig) { config.body = { minConfidence: 0.0001 }; config.hand = { minConfidence: 0.0001 }; res = await testDetect(human, 'samples/in/ai-body.jpg', 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length < 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: sensitive result match'); // test sensitive details face @@ -373,7 +373,7 @@ async function test(Human, inputConfig) { if (!face || face?.box?.length !== 4 || face?.mesh?.length !== 478 || face?.embedding?.length !== 1024 || face?.rotation?.matrix?.length !== 9) { log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length); } else log('state', 'passed: sensitive face result match'); - if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.55 || face.emotion[0].emotion !== 'neutral') log('error', 'failed: sensitive face emotion result mismatch', face?.emotion); + if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.30) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion); else log('state', 'passed: sensitive face emotion result', face?.emotion); // test sensitive details body diff --git a/tsconfig.json b/tsconfig.json index 187e6afe..83db3fb6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,7 +31,7 @@ "noImplicitThis": true, "noPropertyAccessFromIndexSignature": false, "noUncheckedIndexedAccess": false, - "noUnusedLocals": false, + "noUnusedLocals": true, "noUnusedParameters": true, "preserveConstEnums": true, "pretty": true,