From d797c871c74be5d7835da1dd87c9c75e76cf107f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 23 Nov 2021 10:40:40 -0500 Subject: [PATCH] release preview --- CHANGELOG.md | 4 ++++ TODO.md | 10 +++------- src/body/blazeposecoords.ts | 4 ++-- src/face/blazeface.ts | 10 +++++----- src/face/facemesh.ts | 17 ++++++----------- src/util/interpolate.ts | 27 ++++++++++++++------------- test/node.js | 4 +++- test/test-main.js | 6 +++--- 8 files changed, 40 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95a7f50a..09ec1c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ ## Changelog +### **HEAD -> main** 2021/11/23 mandic00@live.com + +- cleanup + ### **2.5.4** 2021/11/22 mandic00@live.com - prototype blazepose detector diff --git a/TODO.md b/TODO.md index 21e51d55..65f21004 100644 --- a/TODO.md +++ b/TODO.md @@ -31,13 +31,6 @@ Feature is automatically disabled in NodeJS without user impact - Backend NodeJS missing kernel op `RotateWithOffset` -### Body Detection - -MoveNet MultiPose model does not work with WASM backend due to missing F32 broadcast implementation - -- Backend WASM missing F32 broadcat implementation - -


## Pending Release Notes @@ -61,3 +54,6 @@ Other: - Improved VSCode out-of-the-box experience - Fix for optional `gear`, `ssrnet`, `mobilefacenet` modules - Fix for Firefox WebGPU compatibility issue +- Fix face detect box scale and rotation +- Fix body interpolation +- Updated `blazepose` implementation diff --git a/src/body/blazeposecoords.ts b/src/body/blazeposecoords.ts index d1c45933..75a9a37d 100644 --- a/src/body/blazeposecoords.ts +++ b/src/body/blazeposecoords.ts @@ -48,9 +48,9 @@ export const connected: Record = { torso: ['leftShoulder', 'rightShoulder', 'rightHip', 'leftHip', 'leftShoulder', 'rightShoulder'], leftArm: ['leftShoulder', 'leftElbow', 'leftWrist', 'leftPalm'], rightArm: ['rightShoulder', 'rightElbow', 'rightWrist', 'rightPalm'], - // leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'], - // rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'], leftEye: ['leftEyeInside', 'leftEye', 'leftEyeOutside'], rightEye: ['rightEyeInside', 'rightEye', 'rightEyeOutside'], mouth: ['leftMouth', 'rightMouth'], + // leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'], + // rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'], }; diff --git a/src/face/blazeface.ts b/src/face/blazeface.ts index 501d22f1..46032df7 100644 --- a/src/face/blazeface.ts +++ b/src/face/blazeface.ts @@ -18,6 +18,8 @@ let anchors: Tensor | null = null; let inputSize = 0; let inputSizeT: Tensor | null = null; +type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; + export const size = () => inputSize; export async function load(config: Config): Promise { @@ -75,7 +77,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) { t.scores = tf.squeeze(t.sigmoid); t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0)); const nms = await t.nms.array() as number[]; - const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = []; + const boxes: Array = []; const scores = await t.scores.data(); for (let i = 0; i < nms.length; i++) { const confidence = scores[nms[i]]; @@ -87,10 +89,8 @@ export async function getBoxes(inputImage: Tensor, config: Config) { b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]); const points = await b.bbox.data(); boxes.push({ - box: { - startPoint: [points[0], points[1]] as Point, - endPoint: [points[2], points[3]] as Point, - }, + startPoint: [points[0], points[1]] as Point, + endPoint: [points[2], points[3]] as Point, landmarks: (await b.landmarks.array()) as Point[], confidence, }); diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index feba4c43..ae86ee58 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -19,8 +19,8 @@ import type { GraphModel, Tensor } from '../tfjs/types'; import type { FaceResult, Point } from '../result'; import type { Config } from '../config'; -type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; -let boxCache: Array = []; +type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; +let boxCache: Array = []; let model: GraphModel | null = null; let inputSize = 0; let skipped = Number.MAX_SAFE_INTEGER; @@ -35,14 +35,9 @@ export async function predict(input: Tensor, config: Config): Promise = []; - const newCache: Array = []; + const newCache: Array = []; let id = 0; for (let i = 0; i < boxCache.length; i++) { let box = boxCache[i]; diff --git a/src/util/interpolate.ts b/src/util/interpolate.ts index 0ad5fc27..c29c9f9f 100644 --- a/src/util/interpolate.ts +++ b/src/util/interpolate.ts @@ -40,26 +40,26 @@ export function calc(newResult: Result, config: Config): Result { } else { for (let i = 0; i < newResult.body.length; i++) { const box = newResult.body[i].box // update box - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as Box; + .map((newBoxCoord, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + newBoxCoord) / bufferedFactor) as Box; const boxRaw = newResult.body[i].boxRaw // update boxRaw - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as Box; + .map((newBoxCoord, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + newBoxCoord) / bufferedFactor) as Box; const keypoints = (newResult.body[i].keypoints // update keypoints - .map((keypoint, j) => ({ - score: keypoint.score, - part: keypoint.part, + .map((newKpt, j) => ({ + score: newKpt.score, + part: newKpt.part, position: [ - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0], - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1], - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[2] || 0) + (keypoint.position[2] || 0)) / bufferedFactor : keypoint.position[2], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[0] || 0) + (newKpt.position[0] || 0)) / bufferedFactor : newKpt.position[0], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[1] || 0) + (newKpt.position[1] || 0)) / bufferedFactor : newKpt.position[1], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[2] || 0) + (newKpt.position[2] || 0)) / bufferedFactor : newKpt.position[2], ], positionRaw: [ - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0], - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1], - bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (keypoint.positionRaw[2] || 0)) / bufferedFactor : keypoint.position[1], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[0] || 0) + (newKpt.positionRaw[0] || 0)) / bufferedFactor : newKpt.position[0], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1], + bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2], ], }))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>; - const annotations: Record = {}; + const annotations: Record = {}; // recreate annotations let coords = { connected: {} }; if (config.body?.modelPath?.includes('efficientpose')) coords = efficientPoseCoords; else if (config.body?.modelPath?.includes('blazepose')) coords = blazePoseCoords; @@ -69,7 +69,8 @@ export function calc(newResult: Result, config: Config): Result { for (let j = 0; j < indexes.length - 1; j++) { const pt0 = keypoints.find((kp) => kp.part === indexes[j]); const pt1 = keypoints.find((kp) => kp.part === indexes[j + 1]); - if (pt0 && pt1 && pt0.score > (config.body.minConfidence || 0) && pt1.score > (config.body.minConfidence || 0)) pt.push([pt0.position, pt1.position]); + // if (pt0 && pt1 && pt0.score > (config.body.minConfidence || 0) && pt1.score > (config.body.minConfidence || 0)) pt.push([pt0.position, pt1.position]); + if (pt0 && pt1) pt.push([pt0.position, pt1.position]); } annotations[name] = pt; } diff --git a/test/node.js b/test/node.js index 341c5bf1..85d0dc15 100644 --- a/test/node.js +++ b/test/node.js @@ -106,7 +106,9 @@ async function testAll() { for (const test of tests) await runTest(test); log.info('all tests complete'); log.info('failed:', { count: failedMessages.length, messages: failedMessages }); - log.info('status:', status); + for (const [test, result] of Object.entries(status)) { + log.info('status:', { test, ...result }); + } } testAll(); diff --git a/test/test-main.js b/test/test-main.js index 737a8096..b55edc04 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -334,7 +334,7 @@ async function test(Human, inputConfig) { res1 = human.similarity(desc1, desc1); res2 = human.similarity(desc1, desc2); res3 = human.similarity(desc1, desc3); - if (res1 < 1 || res2 < 0.50 || res3 < 0.50) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); + if (res1 < 1 || res2 < 0.50 || res3 < 0.45 || res2 > 0.75 || res3 > 0.75) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); // test face matching @@ -373,8 +373,8 @@ async function test(Human, inputConfig) { if (!face || face?.box?.length !== 4 || face?.mesh?.length !== 478 || face?.embedding?.length !== 1024 || face?.rotation?.matrix?.length !== 9) { log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length); } else log('state', 'passed: sensitive face result match'); - if (!face || face?.emotion?.length < 3) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion.length); - else log('state', 'passed: sensitive face emotion result', face?.emotion.length); + if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.55 || face.emotion[0].emotion !== 'neutral') log('error', 'failed: sensitive face emotion result mismatch', face?.emotion); + else log('state', 'passed: sensitive face emotion result', face?.emotion); // test sensitive details body const body = res && res.body ? res.body[0] : null;