From e19c65526c20078e46b42154fbec7913573b5dd9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 1 Jun 2021 07:37:17 -0400 Subject: [PATCH] breaking changes to results.face output properties --- src/draw/draw.ts | 4 ++-- src/face.ts | 11 ++++++----- src/handpose/handpose.ts | 2 +- src/interpolate.ts | 22 +++++++++++++--------- src/result.ts | 35 ++++++++++++++++++----------------- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 99c18987..ffb9240f 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -174,8 +174,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, dra if (localOptions.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions); // silly hack since fillText does not suport new line const labels:string[] = []; - labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`); - if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`); + labels.push(`face confidence: ${Math.trunc(100 * f.score)}%`); + if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}% confident`); // if (f.genderConfidence) labels.push(f.gender); if (f.age) labels.push(`age: ${f.age || ''}`); if (f.iris) labels.push(`distance: ${f.iris}`); diff --git a/src/face.ts b/src/face.ts index 639caeee..7f78799e 100644 --- a/src/face.ts +++ b/src/face.ts @@ -4,6 +4,7 @@ */ import { log, now } from './helpers'; +import * as tf from '../dist/tfjs.esm.js'; import * as facemesh from './blazeface/facemesh'; import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; @@ -159,7 +160,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): parent.analyze('Get Face'); // is something went wrong, skip the face - if (!faces[i].image || faces[i].image.isDisposedInternal) { + if (!faces[i].image || faces[i].image['isDisposedInternal']) { log('Face object is disposed:', faces[i].image); continue; } @@ -210,19 +211,19 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): // combine results faceRes.push({ - id: i, ...faces[i], + id: i, age: descRes.age, gender: descRes.gender, - genderConfidence: descRes.genderConfidence, + genderScore: descRes.genderConfidence, embedding: descRes.descriptor, emotion: emotionRes, iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0, rotation, - tensor: parent.config.face.detector.return ? faces[i].image?.squeeze() : null, + tensor: parent.config.face.detector.return ? tf.squeeze(faces[i].image) : null, }); // dispose original face tensor - faces[i].image?.dispose(); + tf.dispose(faces[i].image); parent.analyze('End Face'); } diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 1f6c3325..8462b16b 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -63,7 +63,7 @@ export async function predict(input, config): Promise { (predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / input.shape[1], ]; } - hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations }); + hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations }); } return hands; } diff --git a/src/interpolate.ts b/src/interpolate.ts index 8b03a355..58386dc4 100644 --- a/src/interpolate.ts +++ b/src/interpolate.ts @@ -70,18 +70,22 @@ export function calc(newResult: Result): Result { .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as [number, number, number, number]; const boxRaw = (newResult.face[i].boxRaw // update boxRaw .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number]; - const matrix = newResult.face[i].rotation.matrix; - const angle = { - roll: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / bufferedFactor, - yaw: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / bufferedFactor, - pitch: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / bufferedFactor, + const rotation: { + matrix: [number, number, number, number, number, number, number, number, number], + angle: { roll: number, yaw: number, pitch: number }, + gaze: { bearing: number, strength: number } + } = { matrix: [0, 0, 0, 0, 0, 0, 0, 0, 0], angle: { roll: 0, yaw: 0, pitch: 0 }, gaze: { bearing: 0, strength: 0 } }; + rotation.matrix = newResult.face[i].rotation?.matrix as [number, number, number, number, number, number, number, number, number]; + rotation.angle = { + roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.roll || 0) + (newResult.face[i].rotation?.angle?.roll || 0)) / bufferedFactor, + yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.yaw || 0) + (newResult.face[i].rotation?.angle?.yaw || 0)) / bufferedFactor, + pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.pitch || 0) + (newResult.face[i].rotation?.angle?.pitch || 0)) / bufferedFactor, }; - const gaze = { + rotation.gaze = { // not fully correct due projection on circle, also causes wrap-around draw on jump from negative to positive - bearing: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.bearing + newResult.face[i].rotation.gaze.bearing) / bufferedFactor, - strength: ((bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / bufferedFactor, + bearing: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze?.bearing || 0) + (newResult.face[i].rotation?.gaze?.bearing || 0)) / bufferedFactor, + strength: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze?.strength || 0) + (newResult.face[i].rotation?.gaze?.strength || 0)) / bufferedFactor, }; - const rotation = { angle, matrix, gaze }; bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values } } diff --git a/src/result.ts b/src/result.ts index b4f13595..3a024fb7 100644 --- a/src/result.ts +++ b/src/result.ts @@ -10,9 +10,9 @@ import { Tensor } from '../dist/tfjs.esm.js'; * * Each result has: * - id: face id number - * - confidence: overal detection confidence value - * - boxConfidence: face box detection confidence value - * - faceConfidence: face keypoints detection confidence value + * - score: overal detection confidence score value + * - boxScore: face box detection confidence score value + * - faceScore: face keypoints detection confidence score value * - box: face bounding box as array of [x, y, width, height], normalized to image resolution * - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1 * - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution @@ -20,7 +20,7 @@ import { Tensor } from '../dist/tfjs.esm.js'; * - annotations: annotated face keypoints as array of annotated face mesh points * - age: age as value * - gender: gender as value - * - genderConfidence: gender detection confidence as value + * - genderScore: gender detection confidence score as value * - emotion: emotions as array of possible emotions with their individual scores * - embedding: facial descriptor as array of numerical elements * - iris: iris distance from current viewpoint as distance value in centimeters for a typical camera @@ -33,25 +33,26 @@ import { Tensor } from '../dist/tfjs.esm.js'; */ export interface Face { id: number - confidence: number, - boxConfidence: number, - faceConfidence: number, + score: number, + boxScore: number, + faceScore: number, box: [number, number, number, number], boxRaw: [number, number, number, number], mesh: Array<[number, number, number]> meshRaw: Array<[number, number, number]> - annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>, - age: number, - gender: string, - genderConfidence: number, - emotion: Array<{ score: number, emotion: string }>, - embedding: Array, - iris: number, - rotation: { + annotations: Record>, + age?: number, + gender?: string, + genderScore?: number, + emotion?: Array<{ score: number, emotion: string }>, + embedding?: Array, + iris?: number, + rotation?: { angle: { roll: number, yaw: number, pitch: number }, matrix: [number, number, number, number, number, number, number, number, number], gaze: { bearing: number, strength: number }, } + image: typeof Tensor; tensor: typeof Tensor, } @@ -86,7 +87,7 @@ export interface Body { * * Each result has: * - id: hand id number - * - confidence: detection confidence score as value + * - score: detection confidence score as value * - box: bounding box: x, y, width, height normalized to input image resolution * - boxRaw: bounding box: x, y, width, height normalized to 0..1 * - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution @@ -94,7 +95,7 @@ export interface Body { */ export interface Hand { id: number, - confidence: number, + score: number, box: [number, number, number, number], boxRaw: [number, number, number, number], keypoints: Array<[number, number, number]>,