ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

release preview

pull/293/head
Vladimir Mandic 2021-11-23 10:40:40 -05:00
parent ec41b72710
commit 690a94f5ea
8 changed files with 40 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -9,6 +9,10 @@
## Changelog
### **HEAD -> main** 2021/11/23 mandic00@live.com
- cleanup
### **2.5.4** 2021/11/22 mandic00@live.com
- prototype blazepose detector

10
TODO.md
View File

@ -31,13 +31,6 @@ Feature is automatically disabled in NodeJS without user impact
- Backend NodeJS missing kernel op `RotateWithOffset`
<https://github.com/tensorflow/tfjs/issues/5473>
### Body Detection
MoveNet MultiPose model does not work with WASM backend due to missing F32 broadcast implementation
- Backend WASM missing F32 broadcat implementation
<https://github.com/tensorflow/tfjs/issues/5516>
<br><hr><br>
## Pending Release Notes
@ -61,3 +54,6 @@ Other:
- Improved VSCode out-of-the-box experience
- Fix for optional `gear`, `ssrnet`, `mobilefacenet` modules
- Fix for Firefox WebGPU compatibility issue
- Fix face detect box scale and rotation
- Fix body interpolation
- Updated `blazepose` implementation

4
src/body/blazeposecoords.ts
View File

@ -48,9 +48,9 @@ export const connected: Record<string, string[]> = {
torso: ['leftShoulder', 'rightShoulder', 'rightHip', 'leftHip', 'leftShoulder', 'rightShoulder'],
leftArm: ['leftShoulder', 'leftElbow', 'leftWrist', 'leftPalm'],
rightArm: ['rightShoulder', 'rightElbow', 'rightWrist', 'rightPalm'],
// leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'],
// rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'],
leftEye: ['leftEyeInside', 'leftEye', 'leftEyeOutside'],
rightEye: ['rightEyeInside', 'rightEye', 'rightEyeOutside'],
mouth: ['leftMouth', 'rightMouth'],
// leftHand: ['leftHand', 'leftPalm', 'leftPinky', 'leftPalm', 'leftIndex', 'leftPalm', 'leftThumb'],
// rightHand: ['rightHand', 'rightPalm', 'rightPinky', 'rightPalm', 'rightIndex', 'rightPalm', 'rightThumb'],
};

10
src/face/blazeface.ts
View File

@ -18,6 +18,8 @@ let anchors: Tensor | null = null;
let inputSize = 0;
let inputSizeT: Tensor | null = null;
type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
export const size = () => inputSize;
export async function load(config: Config): Promise<GraphModel> {
@ -75,7 +77,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
t.scores = tf.squeeze(t.sigmoid);
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
const nms = await t.nms.array() as number[];
const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = [];
const boxes: Array<DetectBox> = [];
const scores = await t.scores.data();
for (let i = 0; i < nms.length; i++) {
const confidence = scores[nms[i]];
@ -87,10 +89,8 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
const points = await b.bbox.data();
boxes.push({
box: {
startPoint: [points[0], points[1]] as Point,
endPoint: [points[2], points[3]] as Point,
},
startPoint: [points[0], points[1]] as Point,
endPoint: [points[2], points[3]] as Point,
landmarks: (await b.landmarks.array()) as Point[],
confidence,
});

17
src/face/facemesh.ts
View File

@ -19,8 +19,8 @@ import type { GraphModel, Tensor } from '../tfjs/types';
import type { FaceResult, Point } from '../result';
import type { Config } from '../config';
type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
let boxCache: Array<BoxCache> = [];
type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
let boxCache: Array<DetectBox> = [];
let model: GraphModel | null = null;
let inputSize = 0;
let skipped = Number.MAX_SAFE_INTEGER;
@ -35,14 +35,9 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
lastTime = now();
boxCache = []; // empty cache
for (const possible of possibleBoxes.boxes) { // extract data from detector
const box: BoxCache = {
startPoint: possible.box.startPoint,
endPoint: possible.box.endPoint,
landmarks: possible.landmarks,
confidence: possible.confidence,
};
const boxScaled = util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor);
const calcFactor = (config.face.detector?.cropFactor || 1.6) * 1400 / (boxScaled.endPoint[0] - boxScaled.startPoint[0] + 1400); // detected face box is not the same size as calculated face box and scale also depends on detected face size
const boxScaled = util.scaleBoxCoordinates(possible, possibleBoxes.scaleFactor);
const detectedWidth = (boxScaled.endPoint[0] - boxScaled.startPoint[0]) / (input.shape[2] || 1000);
const calcFactor = (config.face.detector?.cropFactor || 1.6) / (detectedWidth + 0.75) / 1.34; // detected face box is not the same size as calculated face box and scale also depends on detected face size
const boxEnlarged = util.enlargeBox(boxScaled, calcFactor);
const boxSquared = util.squarifyBox(boxEnlarged);
boxCache.push(boxSquared);
@ -52,7 +47,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
skipped++;
}
const faces: Array<FaceResult> = [];
const newCache: Array<BoxCache> = [];
const newCache: Array<DetectBox> = [];
let id = 0;
for (let i = 0; i < boxCache.length; i++) {
let box = boxCache[i];

27
src/util/interpolate.ts
View File

@ -40,26 +40,26 @@ export function calc(newResult: Result, config: Config): Result {
} else {
for (let i = 0; i < newResult.body.length; i++) {
const box = newResult.body[i].box // update box
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as Box;
.map((newBoxCoord, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + newBoxCoord) / bufferedFactor) as Box;
const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as Box;
.map((newBoxCoord, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + newBoxCoord) / bufferedFactor) as Box;
const keypoints = (newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({
score: keypoint.score,
part: keypoint.part,
.map((newKpt, j) => ({
score: newKpt.score,
part: newKpt.part,
position: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[0] + keypoint.position[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position[1] + keypoint.position[1]) / bufferedFactor : keypoint.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[2] || 0) + (keypoint.position[2] || 0)) / bufferedFactor : keypoint.position[2],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[0] || 0) + (newKpt.position[0] || 0)) / bufferedFactor : newKpt.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[1] || 0) + (newKpt.position[1] || 0)) / bufferedFactor : newKpt.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[2] || 0) + (newKpt.position[2] || 0)) / bufferedFactor : newKpt.position[2],
],
positionRaw: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[0] + keypoint.positionRaw[0]) / bufferedFactor : keypoint.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].positionRaw[1] + keypoint.positionRaw[1]) / bufferedFactor : keypoint.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (keypoint.positionRaw[2] || 0)) / bufferedFactor : keypoint.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[0] || 0) + (newKpt.positionRaw[0] || 0)) / bufferedFactor : newKpt.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2],
],
}))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
const annotations: Record<string, Point[][]> = {};
const annotations: Record<string, Point[][]> = {}; // recreate annotations
let coords = { connected: {} };
if (config.body?.modelPath?.includes('efficientpose')) coords = efficientPoseCoords;
else if (config.body?.modelPath?.includes('blazepose')) coords = blazePoseCoords;
@ -69,7 +69,8 @@ export function calc(newResult: Result, config: Config): Result {
for (let j = 0; j < indexes.length - 1; j++) {
const pt0 = keypoints.find((kp) => kp.part === indexes[j]);
const pt1 = keypoints.find((kp) => kp.part === indexes[j + 1]);
if (pt0 && pt1 && pt0.score > (config.body.minConfidence || 0) && pt1.score > (config.body.minConfidence || 0)) pt.push([pt0.position, pt1.position]);
// if (pt0 && pt1 && pt0.score > (config.body.minConfidence || 0) && pt1.score > (config.body.minConfidence || 0)) pt.push([pt0.position, pt1.position]);
if (pt0 && pt1) pt.push([pt0.position, pt1.position]);
}
annotations[name] = pt;
}

4
test/node.js
View File

@ -106,7 +106,9 @@ async function testAll() {
for (const test of tests) await runTest(test);
log.info('all tests complete');
log.info('failed:', { count: failedMessages.length, messages: failedMessages });
log.info('status:', status);
for (const [test, result] of Object.entries(status)) {
log.info('status:', { test, ...result });
}
}
testAll();

6
test/test-main.js
View File

@ -334,7 +334,7 @@ async function test(Human, inputConfig) {
res1 = human.similarity(desc1, desc1);
res2 = human.similarity(desc1, desc2);
res3 = human.similarity(desc1, desc3);
if (res1 < 1 || res2 < 0.50 || res3 < 0.50) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
if (res1 < 1 || res2 < 0.50 || res3 < 0.45 || res2 > 0.75 || res3 > 0.75) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
// test face matching
@ -373,8 +373,8 @@ async function test(Human, inputConfig) {
if (!face || face?.box?.length !== 4 || face?.mesh?.length !== 478 || face?.embedding?.length !== 1024 || face?.rotation?.matrix?.length !== 9) {
log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length);
} else log('state', 'passed: sensitive face result match');
if (!face || face?.emotion?.length < 3) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion.length);
else log('state', 'passed: sensitive face emotion result', face?.emotion.length);
if (!face || face?.emotion?.length < 1 || face.emotion[0].score < 0.55 || face.emotion[0].emotion !== 'neutral') log('error', 'failed: sensitive face emotion result mismatch', face?.emotion);
else log('state', 'passed: sensitive face emotion result', face?.emotion);
// test sensitive details body
const body = res && res.body ? res.body[0] : null;