From 3a0436bc548c9e6e4fd9f56d8ddfe6ae0fd8f1df Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 3 Nov 2021 16:32:07 -0400 Subject: [PATCH] improve box rescaling for all modules --- CHANGELOG.md | 6 +-- demo/typescript/index.js | 9 +---- demo/typescript/index.ts | 12 +++--- src/face/facemesh.ts | 81 +++++++++++++++++++--------------------- src/face/facemeshutil.ts | 54 ++++++++++++++++----------- src/face/faceres.ts | 7 ++-- src/gear/emotion.ts | 6 ++- src/gesture/gesture.ts | 9 +++-- test/test-main.js | 25 ++++++++----- 9 files changed, 111 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1e549a..91e160cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,9 @@ ## Changelog -### **HEAD -> main** 2021/10/31 mandic00@live.com - - -### **origin/main** 2021/10/31 mandic00@live.com +### **HEAD -> main** 2021/11/02 mandic00@live.com +- refactor predict with execute - patch tfjs type defs - start 2.5 major version - build and docs cleanup diff --git a/demo/typescript/index.js b/demo/typescript/index.js index 55ca3aad..c1f9e7f9 100644 --- a/demo/typescript/index.js +++ b/demo/typescript/index.js @@ -9,12 +9,7 @@ import Human from "../../dist/human.esm.js"; var config = { modelBasePath: "../../models", backend: "humangl", - async: true, - face: { enabled: true }, - body: { enabled: true }, - hand: { enabled: true }, - object: { enabled: false }, - gesture: { enabled: true } + async: true }; var human = new Human(config); human.env.perfadd = false; @@ -39,7 +34,7 @@ var perf = (msg) => { }; async function webCam() { status("starting webcam..."); - const options = { audio: false, video: { facingMode: "user", resizeMode: "none", width: { ideal: document.body.clientWidth } } }; + const options = { audio: false, video: { facingMode: "user", resizeMode: "crop-and-scale", width: { ideal: document.body.clientWidth } } }; const stream = await navigator.mediaDevices.getUserMedia(options); const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); diff --git a/demo/typescript/index.ts b/demo/typescript/index.ts index a55b4286..ca1969f1 100644 --- a/demo/typescript/index.ts +++ b/demo/typescript/index.ts @@ -15,11 +15,11 @@ const config = { modelBasePath: '../../models', backend: 'humangl', async: true, - face: { enabled: true }, - body: { enabled: true }, - hand: { enabled: true }, - object: { enabled: false }, - gesture: { enabled: true }, + // face: { enabled: true, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } }, + // body: { enabled: false }, + // hand: { enabled: false }, + // object: { enabled: false }, + // gesture: { enabled: true }, }; const human = new Human(config); @@ -50,7 +50,7 @@ const perf = (msg) => { async function webCam() { status('starting webcam...'); - const options = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } }; + const options = { audio: false, video: { facingMode: 'user', resizeMode: 'crop-and-scale', width: { ideal: document.body.clientWidth } } }; const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options); const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); }); dom.video.srcObject = stream; diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index cfa62cb2..8795f25f 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -18,48 +18,45 @@ import type { FaceResult, Point } from '../result'; import type { Config } from '../config'; import { env } from '../util/env'; -type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number, faceConfidence?: number | undefined }; +type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; let boxCache: Array = []; let model: GraphModel | null = null; let inputSize = 0; let skipped = Number.MAX_SAFE_INTEGER; let lastTime = 0; -let detectedFaces = 0; +const enlargeFact = 1.6; export async function predict(input: Tensor, config: Config): Promise { // reset cached boxes const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime); const skipFrame = skipped < (config.face.detector?.skipFrames || 0); - if (!config.skipAllowed || !skipTime || !skipFrame || detectedFaces === 0) { - const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector + if (!config.skipAllowed || !skipTime || !skipFrame || boxCache.length === 0) { + const possibleBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector lastTime = now(); boxCache = []; // empty cache - for (const possible of newBoxes.boxes) { // extract data from detector - const startPoint = await possible.box.startPoint.data() as unknown as Point; - const endPoint = await possible.box.endPoint.data() as unknown as Point; - const landmarks = await possible.landmarks.array() as Array; - boxCache.push({ startPoint, endPoint, landmarks, confidence: possible.confidence }); - } - newBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks])); - for (let i = 0; i < boxCache.length; i++) { // enlarge and squarify detected boxes - const scaledBox = util.scaleBoxCoordinates({ startPoint: boxCache[i].startPoint, endPoint: boxCache[i].endPoint }, newBoxes.scaleFactor); - const enlargedBox = util.enlargeBox(scaledBox); - const squarifiedBox = util.squarifyBox(enlargedBox); - boxCache[i] = { ...squarifiedBox, confidence: boxCache[i].confidence, landmarks: boxCache[i].landmarks }; + for (const possible of possibleBoxes.boxes) { // extract data from detector + const box: BoxCache = { + startPoint: await possible.box.startPoint.data() as unknown as Point, + endPoint: await possible.box.endPoint.data() as unknown as Point, + landmarks: await possible.landmarks.array() as Array, + confidence: possible.confidence, + }; + boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact)))); } + possibleBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks])); skipped = 0; } else { skipped++; } - const faces: Array = []; - const newBoxes: Array = []; + const newCache: Array = []; let id = 0; - for (let box of boxCache) { + for (let i = 0; i < boxCache.length; i++) { + let box = boxCache[i]; let angle = 0; let rotationMatrix; - const face: FaceResult = { + const face: FaceResult = { // init face result id: id++, mesh: [], meshRaw: [], @@ -74,16 +71,15 @@ export async function predict(input: Tensor, config: Config): Promise [ ((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()), ((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()), @@ -94,37 +90,36 @@ export async function predict(input: Tensor, config: Config): Promise; // first returned tensor represents facial contours which are already included in the coordinates. - tf.dispose(contours); - const faceConfidence = (await confidence.data())[0] as number; - tf.dispose(confidence); + const faceConfidence = await confidence.data(); + face.faceScore = Math.round(100 * faceConfidence[0]) / 100; const coordsReshaped = tf.reshape(contourCoords, [-1, 3]); let rawCoords = await coordsReshaped.array(); - tf.dispose(contourCoords); - tf.dispose(coordsReshaped); - if (faceConfidence < (config.face.detector?.minConfidence || 1)) { - box.confidence = faceConfidence; // reset confidence of cached box + tf.dispose([contourCoords, coordsReshaped, confidence, contours]); + if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh + box.confidence = face.faceScore; // reset confidence of cached box } else { if (config.face.iris?.enabled) rawCoords = await iris.augmentIris(rawCoords, face.tensor, config, inputSize); // augment results with iris face.mesh = util.transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize); // get processed mesh face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]); - box = { ...util.enlargeBox(util.calculateLandmarksBoundingBox(face.mesh), 1.5), confidence: box.confidence }; // redefine box with mesh calculated one for (const key of Object.keys(coords.meshAnnotations)) face.annotations[key] = coords.meshAnnotations[key].map((index) => face.mesh[index]); // add annotations - if (config.face.detector?.rotation && config.face.mesh.enabled && config.face.description?.enabled && env.kernels.includes('rotatewithoffset')) { // do rotation one more time with mesh keypoints if we want to return perfect image - tf.dispose(face.tensor); // dispose so we can overwrite original face - [angle, rotationMatrix, face.tensor] = util.correctFaceRotation(box, input, inputSize); - } + box = util.squarifyBox(util.enlargeBox(util.calculateLandmarksBoundingBox(face.mesh), enlargeFact)); // redefine box with mesh calculated one face.box = util.getClampedBox(box, input); // update detected box with box around the face mesh face.boxRaw = util.getRawBox(box, input); - face.score = Math.round(100 * faceConfidence || 100 * box.confidence || 0) / 100; - face.faceScore = Math.round(100 * faceConfidence) / 100; - box = { ...util.squarifyBox(box), confidence: box.confidence, faceConfidence }; // updated stored cache values + face.score = face.faceScore; + newCache.push(box); + + // other modules prefer wider crop for a face so we dispose it and do it again + /* + tf.dispose(face.tensor); + face.tensor = config.face.detector?.rotation && config.face.mesh?.enabled && env.kernels.includes('rotatewithoffset') + ? face.tensor = util.correctFaceRotation(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, inputSize)[2] + : face.tensor = util.cutBoxFromImageAndResize(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, [inputSize, inputSize]); + */ } } faces.push(face); - newBoxes.push(box); } - if (config.face.mesh?.enabled) boxCache = newBoxes.filter((a) => a.confidence > (config.face.detector?.minConfidence || 0)); // remove cache entries for detected boxes on low confidence - detectedFaces = faces.length; + boxCache = [...newCache]; // reset cache return faces; } diff --git a/src/face/facemeshutil.ts b/src/face/facemeshutil.ts index 626930a1..f5741501 100644 --- a/src/face/facemeshutil.ts +++ b/src/face/facemeshutil.ts @@ -32,36 +32,39 @@ export const getRawBox = (box, input): Box => (box ? [ export const scaleBoxCoordinates = (box, factor) => { const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]]; const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]]; - return { startPoint, endPoint }; + return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence }; }; export const cutBoxFromImageAndResize = (box, image, cropSize) => { const h = image.shape[1]; const w = image.shape[2]; - return tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize); + const crop = tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize); + const norm = tf.div(crop, 255); + tf.dispose(crop); + return norm; }; -export const enlargeBox = (box, factor = 1.5) => { +export const enlargeBox = (box, factor) => { const center = getBoxCenter(box); const size = getBoxSize(box); const halfSize: [number, number] = [factor * size[0] / 2, factor * size[1] / 2]; - return { startPoint: [center[0] - halfSize[0], center[1] - halfSize[1]] as Point, endPoint: [center[0] + halfSize[0], center[1] + halfSize[1]] as Point, landmarks: box.landmarks }; + return { startPoint: [center[0] - halfSize[0], center[1] - halfSize[1]] as Point, endPoint: [center[0] + halfSize[0], center[1] + halfSize[1]] as Point, landmarks: box.landmarks, confidence: box.confidence }; }; export const squarifyBox = (box) => { const centers = getBoxCenter(box); const size = getBoxSize(box); const halfSize = Math.max(...size) / 2; - return { startPoint: [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)] as Point, endPoint: [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)] as Point, landmarks: box.landmarks }; + return { startPoint: [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)] as Point, endPoint: [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)] as Point, landmarks: box.landmarks, confidence: box.confidence }; }; export const calculateLandmarksBoundingBox = (landmarks) => { const xs = landmarks.map((d) => d[0]); const ys = landmarks.map((d) => d[1]); - return { startPoint: [Math.min(...xs), Math.min(...ys)], endPoint: [Math.max(...xs), Math.max(...ys)], landmarks }; + return { startPoint: [Math.min(...xs), Math.min(...ys)] as Point, endPoint: [Math.max(...xs), Math.max(...ys)] as Point, landmarks }; }; -export const IDENTITY_MATRIX = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; +export const fixedRotationMatrix = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; export const normalizeRadians = (angle) => angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI)); @@ -71,7 +74,7 @@ export const radToDegrees = (rad) => rad * 180 / Math.PI; export const buildTranslationMatrix = (x, y) => [[1, 0, x], [0, 1, y], [0, 0, 1]]; -export const dot = (v1, v2) => { +export const dot = (v1: number[], v2: number[]) => { let product = 0; for (let i = 0; i < v1.length; i++) product += v1[i] * v2[i]; return product; @@ -133,16 +136,17 @@ export function generateAnchors(inputSize) { return anchors; } -export function transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize) { - const boxSize = getBoxSize({ startPoint: box.startPoint, endPoint: box.endPoint }); - const coordsScaled = rawCoords.map((coord) => ([ +export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputSize) { + const boxSize = getBoxSize(box); + const coordsScaled = coordsRaw.map((coord) => ([ // scaled around zero-point boxSize[0] / inputSize * (coord[0] - inputSize / 2), boxSize[1] / inputSize * (coord[1] - inputSize / 2), coord[2] || 0, ])); - const coordsRotationMatrix = (angle !== 0) ? buildRotationMatrix(angle, [0, 0]) : IDENTITY_MATRIX; - const coordsRotated = (angle !== 0) ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled; - const inverseRotationMatrix = (angle !== 0) ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX; + const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); + const coordsRotationMatrix = largeAngle ? buildRotationMatrix(angle, [0, 0]) : fixedRotationMatrix; + const coordsRotated = largeAngle ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled; + const inverseRotationMatrix = largeAngle ? invertTransformMatrix(rotationMatrix) : fixedRotationMatrix; const boxCenter = [...getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1]; return coordsRotated.map((coord) => ([ Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])), @@ -154,13 +158,19 @@ export function transformRawCoords(rawCoords, box, angle, rotationMatrix, inputS export function correctFaceRotation(box, input, inputSize) { const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count) ? coords.meshLandmarks.symmetryLine : coords.blazeFaceLandmarks.symmetryLine; const angle: number = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]); - const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); - const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; - const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node - const rotationMatrix = buildRotationMatrix(-angle, faceCenter); - const cut = cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotated, [inputSize, inputSize]); - const face = tf.div(cut, 255); - tf.dispose(cut); - tf.dispose(rotated); + const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); + let rotationMatrix; + let face; + if (largeAngle) { + const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); + const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; + const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node + rotationMatrix = buildRotationMatrix(-angle, faceCenter); + face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]); + tf.dispose(rotated); + } else { + rotationMatrix = fixedRotationMatrix; + face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]); + } return [angle, rotationMatrix, face]; } diff --git a/src/face/faceres.ts b/src/face/faceres.ts index b784bebd..783919d2 100644 --- a/src/face/faceres.ts +++ b/src/face/faceres.ts @@ -43,13 +43,14 @@ export function enhance(input): Tensor { const tensor = input.image || input.tensor || input; if (!(tensor instanceof tf.Tensor)) return null; // do a tight crop of image and resize it to fit the model - const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right - // const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test if (!model?.inputs[0].shape) return null; // model has no shape so no point continuing + const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); + /* + const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right const crop = (tensor.shape.length === 3) ? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); - + */ /* // just resize to fit the embedding model instead of cropping const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); diff --git a/src/gear/emotion.ts b/src/gear/emotion.ts index 89623918..0a4877b3 100644 --- a/src/gear/emotion.ts +++ b/src/gear/emotion.ts @@ -43,7 +43,11 @@ export async function predict(image: Tensor, config: Config, idx, count) { return new Promise(async (resolve) => { const obj: Array<{ score: number, emotion: string }> = []; if (config.face.emotion?.enabled) { - const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false); + const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0; + const resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false); + // const box = [[0.15, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right + // const resize = tf.image.cropAndResize(image, box, [0], [inputSize, inputSize]); + const [red, green, blue] = tf.split(resize, 3, 3); tf.dispose(resize); // weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index e227b895..0f9a2725 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -100,11 +100,14 @@ export const iris = (res): GestureResult[] => { gestures.push({ iris: i, gesture: 'facing center' }); } - const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].box[2]; const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].box[2]; + const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].box[2]; if (leftIrisCenterX > 0.06 || rightIrisCenterX > 0.06) center = false; - if (leftIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking right' }); - if (rightIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking left' }); + if (leftIrisCenterX > rightIrisCenterX) { // check eye with bigger offset + if (leftIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking right' }); + } else { + if (rightIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking left' }); + } const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].box[3]; const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].box[3]; diff --git a/test/test-main.js b/test/test-main.js index 6c5cec56..146cdb4b 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -188,12 +188,19 @@ async function test(Human, inputConfig) { else log('state', 'passed: warmup none result match'); config.warmup = 'face'; res = await testWarmup(human, 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 8) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: warmup face result match'); config.warmup = 'body'; res = await testWarmup(human, 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 6) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: warmup body result match'); + log('state', 'details:', { + face: { boxScore: res.face[0].boxScore, faceScore: res.face[0].faceScore, age: res.face[0].age, gender: res.face[0].gender, genderScore: res.face[0].genderScore }, + emotion: res.face[0].emotion, + body: { score: res.body[0].score, keypoints: res.body[0].keypoints.length }, + hand: { boxScore: res.hand[0].boxScore, fingerScore: res.hand[0].fingerScore, keypoints: res.hand[0].keypoints.length }, + gestures: res.gesture, + }); // test default config async log('info', 'test default'); @@ -201,8 +208,8 @@ async function test(Human, inputConfig) { config.async = true; config.cacheSensitivity = 0; res = await testDetect(human, 'samples/in/ai-body.jpg', 'default'); - if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default result face mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); - else log('state', 'passed: default result face match'); + if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default result face mismatch', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore); + else log('state', 'passed: default result face match', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore); // test default config sync log('info', 'test sync'); @@ -210,8 +217,8 @@ async function test(Human, inputConfig) { config.async = false; config.cacheSensitivity = 0; res = await testDetect(human, 'samples/in/ai-body.jpg', 'default'); - if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default sync', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); - else log('state', 'passed: default sync'); + if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default sync', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore); + else log('state', 'passed: default sync', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore); // test image processing const img1 = await human.image(null); @@ -240,7 +247,7 @@ async function test(Human, inputConfig) { res1 = human.similarity(desc1, desc1); res2 = human.similarity(desc1, desc2); res3 = human.similarity(desc1, desc3); - if (res1 < 1 || res2 < 0.55 || res3 < 0.5) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); + if (res1 < 1 || res2 < 0.50 || res3 < 0.50) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); // test face matching @@ -271,7 +278,7 @@ async function test(Human, inputConfig) { config.body = { minConfidence: 0.0001 }; config.hand = { minConfidence: 0.0001 }; res = await testDetect(human, 'samples/in/ai-body.jpg', 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 9) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: sensitive result match'); // test sensitive details face @@ -280,7 +287,7 @@ async function test(Human, inputConfig) { log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length); } else log('state', 'passed: sensitive face result match'); if (!face || face?.emotion?.length < 3) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion.length); - else log('state', 'passed: sensitive face emotion result mismatch', face?.emotion.length); + else log('state', 'passed: sensitive face emotion result', face?.emotion.length); // test sensitive details body const body = res && res.body ? res.body[0] : null;