From 714d95f6ed4fad16f0dad33f8f309f229702c452 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 22 May 2021 12:33:19 -0400 Subject: [PATCH] restructure results strong typing --- CHANGELOG.md | 7 +- demo/index.js | 6 +- src/draw/draw.ts | 65 ++++---- src/efficientpose/efficientpose.ts | 6 +- src/face.ts | 27 +--- src/gesture/gesture.ts | 10 +- src/handpose/handpose.ts | 35 ++-- src/human.ts | 6 +- src/object/centernet.ts | 6 +- src/object/nanodet.ts | 6 +- src/posenet/utils.ts | 7 +- src/result.ts | 247 +++++++++++++++++------------ 12 files changed, 226 insertions(+), 202 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78f74ea0..e1b88501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,12 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/05/21 mandic00@live.com + + ### **1.9.1** 2021/05/21 mandic00@live.com - -### **origin/main** 2021/05/20 mandic00@live.com - +- caching improvements - sanitize server input - remove nanodet weights from default distribution - add experimental mb3-centernet object detection diff --git a/demo/index.js b/demo/index.js index 858084a9..dc568476 100644 --- a/demo/index.js +++ b/demo/index.js @@ -10,7 +10,6 @@ let human; const userConfig = { warmup: 'none', - /* backend: 'webgl', async: false, cacheSensitivity: 0, @@ -27,10 +26,9 @@ const userConfig = { }, hand: { enabled: false }, gesture: { enabled: false }, - body: { enabled: false, modelPath: 'posenet.json' }, + body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'blazepose.json' }, object: { enabled: false }, - */ }; // ui options @@ -46,6 +44,7 @@ const ui = { maxFPSframes: 10, // keep fps history for how many frames modelsPreload: true, // preload human models on startup modelsWarmup: true, // warmup human models on startup + buffered: true, // should output be buffered between frames // internal variables busy: false, // internal camera busy flag @@ -54,7 +53,6 @@ const ui = { camera: {}, // internal, holds details of webcam details detectFPS: [], // internal, holds fps values for detection performance drawFPS: [], // internal, holds fps values for draw performance - buffered: false, // should output be buffered between frames drawWarmup: false, // debug only, should warmup image processing be displayed on startup drawThread: null, // internl, perform draw operations in a separate thread detectThread: null, // internl, perform detect operations in a separate thread diff --git a/src/draw/draw.ts b/src/draw/draw.ts index da7b8e67..a1be2646 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -1,5 +1,6 @@ import { TRI468 as triangulation } from '../blazeface/coords'; import { mergeDeep } from '../helpers'; +import type { Result, Face, Body, Hand, Item, Gesture } from '../result'; /** * Draw Options @@ -59,7 +60,7 @@ export const options: DrawOptions = { fillPolygons: false, useDepth: true, useCurves: false, - bufferedOutput: false, + bufferedOutput: true, useRawBoxes: false, calculateHandBox: true, }; @@ -93,14 +94,14 @@ function rect(ctx, x, y, width, height, localOptions) { ctx.stroke(); } -function lines(ctx, points: number[] = [], localOptions) { +function lines(ctx, points: [number, number, number][] = [], localOptions) { if (points === undefined || points.length === 0) return; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); for (const pt of points) { ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; - ctx.lineTo(pt[0], parseInt(pt[1])); + ctx.lineTo(pt[0], Math.round(pt[1])); } ctx.stroke(); if (localOptions.fillPolygons) { @@ -109,7 +110,7 @@ function lines(ctx, points: number[] = [], localOptions) { } } -function curves(ctx, points: number[] = [], localOptions) { +function curves(ctx, points: [number, number, number][] = [], localOptions) { if (points === undefined || points.length === 0) return; if (!localOptions.useCurves || points.length <= 2) { lines(ctx, points, localOptions); @@ -129,7 +130,7 @@ function curves(ctx, points: number[] = [], localOptions) { } } -export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; @@ -156,7 +157,7 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array, d } } -export async function face(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function face(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; @@ -211,24 +212,24 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, draw lines(ctx, points, localOptions); } // iris: array[center, left, top, right, bottom] - if (f.annotations && f.annotations.leftEyeIris) { + if (f.annotations && f.annotations['leftEyeIris']) { ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; ctx.beginPath(); - const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2; - const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2; - ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + const sizeX = Math.abs(f.annotations['leftEyeIris'][3][0] - f.annotations['leftEyeIris'][1][0]) / 2; + const sizeY = Math.abs(f.annotations['leftEyeIris'][4][1] - f.annotations['leftEyeIris'][2][1]) / 2; + ctx.ellipse(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); if (localOptions.fillPolygons) { ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; ctx.fill(); } } - if (f.annotations && f.annotations.rightEyeIris) { + if (f.annotations && f.annotations['rightEyeIris']) { ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; ctx.beginPath(); - const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2; - const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2; - ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + const sizeX = Math.abs(f.annotations['rightEyeIris'][3][0] - f.annotations['rightEyeIris'][1][0]) / 2; + const sizeY = Math.abs(f.annotations['rightEyeIris'][4][1] - f.annotations['rightEyeIris'][2][1]) / 2; + ctx.ellipse(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); if (localOptions.fillPolygons) { ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; @@ -241,7 +242,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, draw } const lastDrawnPose:any[] = []; -export async function body(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function body(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; @@ -249,20 +250,22 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw if (!ctx) return; ctx.lineJoin = 'round'; for (let i = 0; i < result.length; i++) { - // result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5); if (!lastDrawnPose[i] && localOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; ctx.strokeStyle = localOptions.color; ctx.fillStyle = localOptions.color; ctx.lineWidth = localOptions.lineWidth; ctx.font = localOptions.font; - if (localOptions.drawBoxes) { + if (localOptions.drawBoxes && result[i].box && result[i].box?.length === 4) { + // @ts-ignore box may not exist rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions); if (localOptions.drawLabels) { if (localOptions.shadowColor && localOptions.shadowColor !== '') { ctx.fillStyle = localOptions.shadowColor; + // @ts-ignore box may not exist ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); } ctx.fillStyle = localOptions.labelColor; + // @ts-ignore box may not exist ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); } } @@ -361,7 +364,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw } } -export async function hand(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function hand(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; @@ -415,12 +418,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array, draw ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); }; ctx.font = localOptions.font; - addHandLabel(h.annotations.indexFinger, 'index'); - addHandLabel(h.annotations.middleFinger, 'middle'); - addHandLabel(h.annotations.ringFinger, 'ring'); - addHandLabel(h.annotations.pinky, 'pinky'); - addHandLabel(h.annotations.thumb, 'thumb'); - addHandLabel(h.annotations.palmBase, 'palm'); + addHandLabel(h.annotations['indexFinger'], 'index'); + addHandLabel(h.annotations['middleFinger'], 'middle'); + addHandLabel(h.annotations['ringFinger'], 'ring'); + addHandLabel(h.annotations['pinky'], 'pinky'); + addHandLabel(h.annotations['thumb'], 'thumb'); + addHandLabel(h.annotations['palmBase'], 'palm'); } if (localOptions.drawPolygons) { const addHandLine = (part) => { @@ -434,17 +437,17 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array, draw } }; ctx.lineWidth = localOptions.lineWidth; - addHandLine(h.annotations.indexFinger); - addHandLine(h.annotations.middleFinger); - addHandLine(h.annotations.ringFinger); - addHandLine(h.annotations.pinky); - addHandLine(h.annotations.thumb); + addHandLine(h.annotations['indexFinger']); + addHandLine(h.annotations['middleFinger']); + addHandLine(h.annotations['ringFinger']); + addHandLine(h.annotations['pinky']); + addHandLine(h.annotations['thumb']); // addPart(h.annotations.palmBase); } } } -export async function object(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function object(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; @@ -479,7 +482,7 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE outCtx?.drawImage(inCanvas, 0, 0); } -export async function all(inCanvas: HTMLCanvasElement, result:any, drawOptions?: DrawOptions) { +export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index c891e956..0e89a7f9 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -1,5 +1,6 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import { Body } from '../result'; let model; let keypoints: Array = []; @@ -37,8 +38,7 @@ function max2d(inputs, minScore) { }); } -export async function predict(image, config) { - if (!model) return null; +export async function predict(image, config): Promise { if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) { skipped++; return keypoints; @@ -87,6 +87,6 @@ export async function predict(image, config) { keypoints = parts; } const score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); - resolve([{ score, keypoints }]); + resolve([{ id: 0, score, keypoints }]); }); } diff --git a/src/face.ts b/src/face.ts index 86deec91..2fee4ac2 100644 --- a/src/face.ts +++ b/src/face.ts @@ -1,10 +1,8 @@ import { log, now } from './helpers'; -import * as tf from '../dist/tfjs.esm.js'; import * as facemesh from './blazeface/facemesh'; import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; - -type Tensor = typeof tf.Tensor; +import { Face } from './result'; const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number] } => { // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars @@ -107,27 +105,7 @@ export const detectFace = async (parent, input): Promise => { let emotionRes; let embeddingRes; let descRes; - const faceRes: Array<{ - confidence: number, - boxConfidence: number, - faceConfidence: number, - box: [number, number, number, number], - mesh: Array<[number, number, number]> - meshRaw: Array<[number, number, number]> - boxRaw: [number, number, number, number], - annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>, - age: number, - gender: string, - genderConfidence: number, - emotion: string, - embedding: number[], - iris: number, - rotation: { - angle: { pitch: number, yaw: number, roll: number }, - matrix: [number, number, number, number, number, number, number, number, number] - }, - tensor: Tensor, - }> = []; + const faceRes: Array = []; parent.state = 'run:face'; timeStamp = now(); const faces = await facemesh.predict(input, parent.config); @@ -189,6 +167,7 @@ export const detectFace = async (parent, input): Promise => { // combine results faceRes.push({ + id: i, ...faces[i], age: descRes.age, gender: descRes.gender, diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index db9d063d..2c6fd5ee 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -1,4 +1,6 @@ -export const body = (res) => { +import { Gesture } from '../result'; + +export const body = (res): Gesture[] => { if (!res) return []; const gestures: Array<{ body: number, gesture: string }> = []; for (let i = 0; i < res.length; i++) { @@ -18,7 +20,7 @@ export const body = (res) => { return gestures; }; -export const face = (res) => { +export const face = (res): Gesture[] => { if (!res) return []; const gestures: Array<{ face: number, gesture: string }> = []; for (let i = 0; i < res.length; i++) { @@ -39,7 +41,7 @@ export const face = (res) => { return gestures; }; -export const iris = (res) => { +export const iris = (res): Gesture[] => { if (!res) return []; const gestures: Array<{ iris: number, gesture: string }> = []; for (let i = 0; i < res.length; i++) { @@ -77,7 +79,7 @@ export const iris = (res) => { return gestures; }; -export const hand = (res) => { +export const hand = (res): Gesture[] => { if (!res) return []; const gestures: Array<{ hand: number, gesture: string }> = []; for (let i = 0; i < res.length; i++) { diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 265f558d..9499cfbf 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -2,6 +2,7 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; import * as handpipeline from './handpipeline'; +import { Hand } from '../result'; const meshAnnotations = { thumb: [1, 2, 3, 4], @@ -16,30 +17,30 @@ let handDetectorModel; let handPoseModel; let handPipeline; -export async function predict(input, config) { +export async function predict(input, config): Promise { const predictions = await handPipeline.estimateHands(input, config); if (!predictions) return []; - const hands: Array<{ confidence: number, box: any, boxRaw: any, landmarks: any, annotations: any }> = []; - for (const prediction of predictions) { + const hands: Array = []; + for (let i = 0; i < predictions.length; i++) { const annotations = {}; - if (prediction.landmarks) { + if (predictions[i].landmarks) { for (const key of Object.keys(meshAnnotations)) { - annotations[key] = meshAnnotations[key].map((index) => prediction.landmarks[index]); + annotations[key] = meshAnnotations[key].map((index) => predictions[i].landmarks[index]); } } - const box = prediction.box ? [ - Math.max(0, prediction.box.topLeft[0]), - Math.max(0, prediction.box.topLeft[1]), - Math.min(input.shape[2], prediction.box.bottomRight[0]) - Math.max(0, prediction.box.topLeft[0]), - Math.min(input.shape[1], prediction.box.bottomRight[1]) - Math.max(0, prediction.box.topLeft[1]), - ] : []; - const boxRaw = [ - (prediction.box.topLeft[0]) / input.shape[2], - (prediction.box.topLeft[1]) / input.shape[1], - (prediction.box.bottomRight[0] - prediction.box.topLeft[0]) / input.shape[2], - (prediction.box.bottomRight[1] - prediction.box.topLeft[1]) / input.shape[1], + const box: [number, number, number, number] = predictions[i].box ? [ + Math.max(0, predictions[i].box.topLeft[0]), + Math.max(0, predictions[i].box.topLeft[1]), + Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0]), + Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1]), + ] : [0, 0, 0, 0]; + const boxRaw: [number, number, number, number] = [ + (predictions[i].box.topLeft[0]) / input.shape[2], + (predictions[i].box.topLeft[1]) / input.shape[1], + (predictions[i].box.bottomRight[0] - predictions[i].box.topLeft[0]) / input.shape[2], + (predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / input.shape[1], ]; - hands.push({ confidence: Math.round(100 * prediction.confidence) / 100, box, boxRaw, landmarks: prediction.landmarks, annotations }); + hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, landmarks: predictions[i].landmarks, annotations }); } return hands; } diff --git a/src/human.ts b/src/human.ts index 48cb64ad..89ec09db 100644 --- a/src/human.ts +++ b/src/human.ts @@ -23,7 +23,7 @@ import * as app from '../package.json'; export type Tensor = typeof tf.Tensor; export type { Config } from './config'; -export type { Result } from './result'; +export type { Result, Face, Hand, Body, Item, Gesture } from './result'; export type { DrawOptions } from './draw/draw'; /** Defines all possible input types for **Human** detection */ @@ -530,7 +530,7 @@ export class Human { this.perf.total = Math.trunc(now() - timeStart); this.state = 'idle'; - const result = { + const res = { face: faceRes, body: bodyRes, hand: handRes, @@ -540,7 +540,7 @@ export class Human { canvas: process.canvas, }; // log('Result:', result); - resolve(result); + resolve(res); }); } diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 364db92d..db1283db 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -1,9 +1,10 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; +import { Item } from '../result'; let model; -let last: Array<{}> = []; +let last: Item[] = []; let skipped = Number.MAX_SAFE_INTEGER; export async function load(config) { @@ -58,8 +59,7 @@ async function process(res, inputSize, outputShape, config) { return results; } -export async function predict(image, config) { - if (!model) return null; +export async function predict(image, config): Promise { if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) { skipped++; return last; diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 8bee21d6..6531ae14 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -1,9 +1,10 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; +import { Item } from '../result'; let model; -let last: Array<{}> = []; +let last: Array = []; let skipped = Number.MAX_SAFE_INTEGER; const scaleBox = 2.5; // increase box size @@ -95,8 +96,7 @@ async function process(res, inputSize, outputShape, config) { return results; } -export async function predict(image, config) { - if (!model) return null; +export async function predict(image, config): Promise { if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) { skipped++; return last; diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index 881cb1fd..2423d791 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -30,8 +30,11 @@ export function getBoundingBox(keypoints) { } export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) { - const scalePose = (pose, scaleY, scaleX) => ({ + const scaleY = height / inputResolutionHeight; + const scaleX = width / inputResolutionWidth; + const scalePose = (pose) => ({ score: pose.score, + bowRaw: [pose.box[0] / inputResolutionWidth, pose.box[1] / inputResolutionHeight, pose.box[2] / inputResolutionWidth, pose.box[3] / inputResolutionHeight], box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)], keypoints: pose.keypoints.map(({ score, part, position }) => ({ score, @@ -39,7 +42,7 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) }, })), }); - const scaledPoses = poses.map((pose) => scalePose(pose, height / inputResolutionHeight, width / inputResolutionWidth)); + const scaledPoses = poses.map((pose) => scalePose(pose)); return scaledPoses; } diff --git a/src/result.ts b/src/result.ts index 4902feaa..84f5df00 100644 --- a/src/result.ts +++ b/src/result.ts @@ -3,114 +3,151 @@ * * Contains all possible detection results */ -export interface Result { - /** Face results - * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models - * Some values may be null if specific model is not enabled - * - * Array of individual results with one object per detected face - * Each result has: - * - overal detection confidence value - * - box detection confidence value - * - mesh detection confidence value - * - box as array of [x, y, width, height], normalized to image resolution - * - boxRaw as array of [x, y, width, height], normalized to range 0..1 - * - mesh as array of [x, y, z] points of face mesh, normalized to image resolution - * - meshRaw as array of [x, y, z] points of face mesh, normalized to range 0..1 - * - annotations as array of annotated face mesh points - * - age as value - * - gender as value - * - genderConfidence as value - * - emotion as array of possible emotions with their individual scores - * - iris as distance value - * - angle as object with values for roll, yaw and pitch angles - * - tensor as Tensor object which contains detected face - */ - face: Array<{ - confidence: number, - boxConfidence: number, - faceConfidence: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - mesh: Array<[number, number, number]> - meshRaw: Array<[number, number, number]> - annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>, - age: number, - gender: string, - genderConfidence: number, - emotion: Array<{ score: number, emotion: string }>, - embedding: Array, - iris: number, - rotation: { - angle: { roll: number, yaw: number, pitch: number }, - matrix: Array<[number, number, number, number, number, number, number, number, number]> - } - tensor: any, - }>, - /** Body results - * - * Array of individual results with one object per detected body - * Each results has: - * - body id number - * - body part name - * - part position with x,y,z coordinates - * - body part score value - * - body part presence value - */ - body: Array<{ - id: number, + +/** Face results + * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models + * Some values may be null if specific model is not enabled + * + * Array of individual results with one object per detected face + * Each result has: + * - id: face number + * - confidence: overal detection confidence value + * - boxConfidence: face box detection confidence value + * - faceConfidence: face keypoints detection confidence value + * - box: face bounding box as array of [x, y, width, height], normalized to image resolution + * - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1 + * - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution + * - meshRaw: face keypoints as array of [x, y, z] points of face mesh, normalized to range 0..1 + * - annotations: annotated face keypoints as array of annotated face mesh points + * - age: age as value + * - gender: gender as value + * - genderConfidence: gender detection confidence as value + * - emotion: emotions as array of possible emotions with their individual scores + * - embedding: facial descriptor as array of numerical elements + * - iris: iris distance from current viewpoint as distance value + * - rotation: face rotiation that contains both angles and matrix used for 3d transformations + * - angle: face angle as object with values for roll, yaw and pitch angles + * - matrix: 3d transofrmation matrix as array of numeric values + * - tensor: face tensor as Tensor object which contains detected face + */ +export interface Face { + id: number + confidence: number, + boxConfidence: number, + faceConfidence: number, + box: [number, number, number, number], + boxRaw: [number, number, number, number], + mesh: Array<[number, number, number]> + meshRaw: Array<[number, number, number]> + annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>, + age: number, + gender: string, + genderConfidence: number, + emotion: Array<{ score: number, emotion: string }>, + embedding: Array, + iris: number, + rotation: { + angle: { roll: number, yaw: number, pitch: number }, + matrix: [number, number, number, number, number, number, number, number, number], + } + tensor: any, +} + +/** Body results + * + * Array of individual results with one object per detected body + * Each results has: + * - id:body id number + * - score: overall detection score + * - box: bounding box: x, y, width, height normalized to input image resolution + * - boxRaw: bounding box: x, y, width, height normalized to 0..1 + * - keypoints: array of keypoints + * - part: body part name + * - position: body part position with x,y,z coordinates + * - score: body part score value + * - presence: body part presence value + */ + +export interface Body { + id: number, + score: number, + box?: [x: number, y: number, width: number, height: number], + boxRaw?: [x: number, y: number, width: number, height: number], + keypoints: Array<{ part: string, position: { x: number, y: number, z: number }, score: number, - presence: number }>, - /** Hand results - * - * Array of individual results with one object per detected hand - * Each result has: - * - confidence as value - * - box as array of [x, y, width, height], normalized to image resolution - * - boxRaw as array of [x, y, width, height], normalized to range 0..1 - * - landmarks as array of [x, y, z] points of hand, normalized to image resolution - * - annotations as array of annotated face landmark points - */ - hand: Array<{ - confidence: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - landmarks: Array<[number, number, number]>, - annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>, - }>, - /** Gesture results - * - * Array of individual results with one object per detected gesture - * Each result has: - * - part: part name and number where gesture was detected: face, iris, body, hand - * - gesture: gesture detected - */ - gesture: Array< - { 'face': number, gesture: string } | { 'iris': number, gesture: string } | { 'body': number, gesture: string } | { 'hand': number, gesture: string } - >, - /** Object results - * - * Array of individual results with one object per detected gesture - * Each result has: - * - score as value - * - label as detected class name - * - center as array of [x, y], normalized to image resolution - * - centerRaw as array of [x, y], normalized to range 0..1 - * - box as array of [x, y, width, height], normalized to image resolution - * - boxRaw as array of [x, y, width, height], normalized to range 0..1 - */ - object: Array<{ - score: number, - strideSize: number, - class: number, - label: string, - center: number[], - centerRaw: number[], - box: number[], - boxRaw: number[], - }>, + presence: number, + }> +} + +/** Hand results + * + * Array of individual results with one object per detected hand + * Each result has: + * - confidence as value + * - box as array of [x, y, width, height], normalized to image resolution + * - boxRaw as array of [x, y, width, height], normalized to range 0..1 + * - landmarks as array of [x, y, z] points of hand, normalized to image resolution + * - annotations as array of annotated face landmark points + */ +export interface Hand { + id: number, + confidence: number, + box: [number, number, number, number], + boxRaw: [number, number, number, number], + landmarks: Array<[number, number, number]>, + // annotations: Array<{ part: string, points: Array<[number, number, number]> }>, + // annotations: Annotations, + annotations: Record }>>, +} + +/** Object results +* +* Array of individual results with one object per detected gesture +* Each result has: +* - score as value +* - label as detected class name +* - center as array of [x, y], normalized to image resolution +* - centerRaw as array of [x, y], normalized to range 0..1 +* - box as array of [x, y, width, height], normalized to image resolution +* - boxRaw as array of [x, y, width, height], normalized to range 0..1 +*/ +export interface Item { + score: number, + strideSize?: number, + class: number, + label: string, + center?: number[], + centerRaw?: number[], + box: number[], + boxRaw: number[], +} + +/** Gesture results + * + * Array of individual results with one object per detected gesture + * Each result has: + * - part: part name and number where gesture was detected: face, iris, body, hand + * - gesture: gesture detected + */ +export type Gesture = + { 'face': number, gesture: string } + | { 'iris': number, gesture: string } + | { 'body': number, gesture: string } + | { 'hand': number, gesture: string } + +export interface Result { + /** {@link Face}: detection & analysis results */ + face: Array, + /** {@link Body}: detection & analysis results */ + body: Array, + /** {@link Hand}: detection & analysis results */ + hand: Array, + /** {@link Gesture}: detection & analysis results */ + gesture: Array, + /** {@link Object}: detection & analysis results */ + object: Array performance: { any }, canvas: OffscreenCanvas | HTMLCanvasElement, }