ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

breaking changes to results.face output properties

pull/356/head
Vladimir Mandic 2021-06-01 07:37:17 -04:00
parent 3708732d1a
commit e8cb3a361e
5 changed files with 40 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/draw/draw.ts
View File

@ -174,8 +174,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
if (localOptions.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions);
// silly hack since fillText does not suport new line
const labels:string[] = [];
labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`);
if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`);
labels.push(`face confidence: ${Math.trunc(100 * f.score)}%`);
if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}% confident`);
// if (f.genderConfidence) labels.push(f.gender);
if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`distance: ${f.iris}`);

11
src/face.ts
View File

@ -4,6 +4,7 @@
*/
import { log, now } from './helpers';
import * as tf from '../dist/tfjs.esm.js';
import * as facemesh from './blazeface/facemesh';
import * as emotion from './emotion/emotion';
import * as faceres from './faceres/faceres';
@ -159,7 +160,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
parent.analyze('Get Face');
// is something went wrong, skip the face
if (!faces[i].image || faces[i].image.isDisposedInternal) {
if (!faces[i].image || faces[i].image['isDisposedInternal']) {
log('Face object is disposed:', faces[i].image);
continue;
}
@ -210,19 +211,19 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
// combine results
faceRes.push({
id: i,
...faces[i],
id: i,
age: descRes.age,
gender: descRes.gender,
genderConfidence: descRes.genderConfidence,
genderScore: descRes.genderConfidence,
embedding: descRes.descriptor,
emotion: emotionRes,
iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0,
rotation,
tensor: parent.config.face.detector.return ? faces[i].image?.squeeze() : null,
tensor: parent.config.face.detector.return ? tf.squeeze(faces[i].image) : null,
});
// dispose original face tensor
faces[i].image?.dispose();
tf.dispose(faces[i].image);
parent.analyze('End Face');
}

2
src/handpose/handpose.ts
View File

@ -63,7 +63,7 @@ export async function predict(input, config): Promise<Hand[]> {
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / input.shape[1],
];
}
hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations });
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations });
}
return hands;
}

22
src/interpolate.ts
View File

@ -70,18 +70,22 @@ export function calc(newResult: Result): Result {
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
const boxRaw = (newResult.face[i].boxRaw // update boxRaw
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / bufferedFactor,
yaw: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / bufferedFactor,
pitch: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / bufferedFactor,
const rotation: {
matrix: [number, number, number, number, number, number, number, number, number],
angle: { roll: number, yaw: number, pitch: number },
gaze: { bearing: number, strength: number }
} = { matrix: [0, 0, 0, 0, 0, 0, 0, 0, 0], angle: { roll: 0, yaw: 0, pitch: 0 }, gaze: { bearing: 0, strength: 0 } };
rotation.matrix = newResult.face[i].rotation?.matrix as [number, number, number, number, number, number, number, number, number];
rotation.angle = {
roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.roll || 0) + (newResult.face[i].rotation?.angle?.roll || 0)) / bufferedFactor,
yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.yaw || 0) + (newResult.face[i].rotation?.angle?.yaw || 0)) / bufferedFactor,
pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.pitch || 0) + (newResult.face[i].rotation?.angle?.pitch || 0)) / bufferedFactor,
};
const gaze = {
rotation.gaze = {
// not fully correct due projection on circle, also causes wrap-around draw on jump from negative to positive
bearing: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.bearing + newResult.face[i].rotation.gaze.bearing) / bufferedFactor,
strength: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / bufferedFactor,
bearing: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze?.bearing || 0) + (newResult.face[i].rotation?.gaze?.bearing || 0)) / bufferedFactor,
strength: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze?.strength || 0) + (newResult.face[i].rotation?.gaze?.strength || 0)) / bufferedFactor,
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
}
}

35
src/result.ts
View File

@ -10,9 +10,9 @@ import { Tensor } from '../dist/tfjs.esm.js';
*
* Each result has:
* - id: face id number
* - confidence: overal detection confidence value
* - boxConfidence: face box detection confidence value
* - faceConfidence: face keypoints detection confidence value
* - score: overal detection confidence score value
* - boxScore: face box detection confidence score value
* - faceScore: face keypoints detection confidence score value
* - box: face bounding box as array of [x, y, width, height], normalized to image resolution
* - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1
* - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution
@ -20,7 +20,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - annotations: annotated face keypoints as array of annotated face mesh points
* - age: age as value
* - gender: gender as value
* - genderConfidence: gender detection confidence as value
* - genderScore: gender detection confidence score as value
* - emotion: emotions as array of possible emotions with their individual scores
* - embedding: facial descriptor as array of numerical elements
* - iris: iris distance from current viewpoint as distance value in centimeters for a typical camera
@ -33,25 +33,26 @@ import { Tensor } from '../dist/tfjs.esm.js';
*/
export interface Face {
id: number
confidence: number,
boxConfidence: number,
faceConfidence: number,
score: number,
boxScore: number,
faceScore: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
mesh: Array<[number, number, number]>
meshRaw: Array<[number, number, number]>
annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
age: number,
gender: string,
genderConfidence: number,
emotion: Array<{ score: number, emotion: string }>,
embedding: Array<number>,
iris: number,
rotation: {
annotations: Record<string, Array<[number, number, number]>>,
age?: number,
gender?: string,
genderScore?: number,
emotion?: Array<{ score: number, emotion: string }>,
embedding?: Array<number>,
iris?: number,
rotation?: {
angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { bearing: number, strength: number },
}
image: typeof Tensor;
tensor: typeof Tensor,
}
@ -86,7 +87,7 @@ export interface Body {
*
* Each result has:
* - id: hand id number
* - confidence: detection confidence score as value
* - score: detection confidence score as value
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
* - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution
@ -94,7 +95,7 @@ export interface Body {
*/
export interface Hand {
id: number,
confidence: number,
score: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
keypoints: Array<[number, number, number]>,