From 54c1dfb37a7982e0a10a3790130f41937180feac Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 11 Sep 2021 23:54:35 -0400 Subject: [PATCH] redefine config and result interfaces --- CHANGELOG.md | 1 + src/config.ts | 337 +++++++++++++++-------------- src/draw/draw.ts | 14 +- src/efficientpose/efficientpose.ts | 10 +- src/emotion/emotion.ts | 10 +- src/face.ts | 6 +- src/faceres/faceres.ts | 10 +- src/gesture/gesture.ts | 10 +- src/handpose/handpose.ts | 18 +- src/human.ts | 61 +++--- src/image/image.ts | 8 +- src/interpolate.ts | 16 +- src/movenet/movenet.ts | 8 +- src/object/centernet.ts | 12 +- src/object/nanodet.ts | 12 +- src/persons.ts | 8 +- src/posenet/utils.ts | 4 +- src/result.ts | 42 ++-- 18 files changed, 299 insertions(+), 288 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe60ec59..15ae2a80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### **HEAD -> main** 2021/09/11 mandic00@live.com +- start using partial definitions - implement event emitters - fix iife loader - simplify dependencies diff --git a/src/config.ts b/src/config.ts index 9e65f36e..434ef678 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,9 +8,177 @@ * @typedef Config */ +/** Controlls and configures all face-specific options: + * - face detection, face mesh detection, age, gender, emotion detection and face description + * Parameters: + * - enabled: true/false + * - modelPath: path for each of face models + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance + * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles + * - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor +*/ +export interface FaceConfig { + enabled: boolean, + detector: { + modelPath: string, + rotation: boolean, + maxDetected: number, + skipFrames: number, + minConfidence: number, + iouThreshold: number, + return: boolean, + }, + mesh: { + enabled: boolean, + modelPath: string, + }, + iris: { + enabled: boolean, + modelPath: string, + }, + description: { + enabled: boolean, + modelPath: string, + skipFrames: number, + minConfidence: number, + }, + emotion: { + enabled: boolean, + minConfidence: number, + skipFrames: number, + modelPath: string, + }, +} + +/** Controlls and configures all body detection specific options + * - enabled: true/false + * - modelPath: body pose model, can be absolute path or relative to modelBasePath + * - minConfidence: threshold for discarding a prediction + * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance +*/ +export interface BodyConfig { + enabled: boolean, + modelPath: string, + maxDetected: number, + minConfidence: number, + skipFrames: number, +} + +/** Controlls and configures all hand detection specific options + * - enabled: true/false + * - landmarks: detect hand landmarks or just hand boundary box + * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance + * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted +*/ +export interface HandConfig { + enabled: boolean, + rotation: boolean, + skipFrames: number, + minConfidence: number, + iouThreshold: number, + maxDetected: number, + landmarks: boolean, + detector: { + modelPath: string, + }, + skeleton: { + modelPath: string, + }, +} + +/** Controlls and configures all object detection specific options + * - enabled: true/false + * - modelPath: object detection model, can be absolute path or relative to modelBasePath + * - minConfidence: minimum score that detection must have to return as valid object + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxDetected: maximum number of detections to return +*/ +export interface ObjectConfig { + enabled: boolean, + modelPath: string, + minConfidence: number, + iouThreshold: number, + maxDetected: number, + skipFrames: number, +} + +/** Controlls and configures all body segmentation module + * removes background from input containing person + * if segmentation is enabled it will run as preprocessing task before any other model + * alternatively leave it disabled and use it on-demand using human.segmentation method which can + * remove background or replace it with user-provided background + * + * - enabled: true/false + * - modelPath: object detection model, can be absolute path or relative to modelBasePath +*/ +export interface SegmentationConfig { + enabled: boolean, + modelPath: string, +} + +/** Run input through image filters before inference + * - image filters run with near-zero latency as they are executed on the GPU +*/ +export interface FilterConfig { + enabled: boolean, + /** Resize input width + * - if both width and height are set to 0, there is no resizing + * - if just one is set, second one is scaled automatically + * - if both are set, values are used as-is + */ + width: number, + /** Resize input height + * - if both width and height are set to 0, there is no resizing + * - if just one is set, second one is scaled automatically + * - if both are set, values are used as-is + */ + height: number, + /** Return processed canvas imagedata in result */ + return: boolean, + /** Flip input as mirror image */ + flip: boolean, + /** Range: -1 (darken) to 1 (lighten) */ + brightness: number, + /** Range: -1 (reduce contrast) to 1 (increase contrast) */ + contrast: number, + /** Range: 0 (no sharpening) to 1 (maximum sharpening) */ + sharpness: number, + /** Range: 0 (no blur) to N (blur radius in pixels) */ + blur: number + /** Range: -1 (reduce saturation) to 1 (increase saturation) */ + saturation: number, + /** Range: 0 (no change) to 360 (hue rotation in degrees) */ + hue: number, + /** Image negative */ + negative: boolean, + /** Image sepia colors */ + sepia: boolean, + /** Image vintage colors */ + vintage: boolean, + /** Image kodachrome colors */ + kodachrome: boolean, + /** Image technicolor colors */ + technicolor: boolean, + /** Image polaroid camera effect */ + polaroid: boolean, + /** Range: 0 (no pixelate) to N (number of pixels to pixelate) */ + pixelate: number, +} + +/** Controlls gesture detection */ +export interface GestureConfig { + enabled: boolean, +} + export interface Config { /** Backend used for TFJS operations */ - backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu' | null | string, + // backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu' | null, + backend: string; /** Path to *.wasm files if backend is set to `wasm` */ wasmPath: string, @@ -25,7 +193,8 @@ export interface Config { * - warmup pre-initializes all models for faster inference but can take significant time on startup * - only used for `webgl` and `humangl` backends */ - warmup: 'none' | 'face' | 'full' | 'body' | string, + // warmup: 'none' | 'face' | 'full' | 'body' | string, + warmup: string; /** Base model path (typically starting with file://, http:// or https://) for all models * - individual modelPath values are relative to this path @@ -47,170 +216,20 @@ export interface Config { /** Run input through image filters before inference * - image filters run with near-zero latency as they are executed on the GPU */ - filter: { - enabled: boolean, - /** Resize input width - * - if both width and height are set to 0, there is no resizing - * - if just one is set, second one is scaled automatically - * - if both are set, values are used as-is - */ - width: number, - /** Resize input height - * - if both width and height are set to 0, there is no resizing - * - if just one is set, second one is scaled automatically - * - if both are set, values are used as-is - */ - height: number, - /** Return processed canvas imagedata in result */ - return: boolean, - /** Flip input as mirror image */ - flip: boolean, - /** Range: -1 (darken) to 1 (lighten) */ - brightness: number, - /** Range: -1 (reduce contrast) to 1 (increase contrast) */ - contrast: number, - /** Range: 0 (no sharpening) to 1 (maximum sharpening) */ - sharpness: number, - /** Range: 0 (no blur) to N (blur radius in pixels) */ - blur: number - /** Range: -1 (reduce saturation) to 1 (increase saturation) */ - saturation: number, - /** Range: 0 (no change) to 360 (hue rotation in degrees) */ - hue: number, - /** Image negative */ - negative: boolean, - /** Image sepia colors */ - sepia: boolean, - /** Image vintage colors */ - vintage: boolean, - /** Image kodachrome colors */ - kodachrome: boolean, - /** Image technicolor colors */ - technicolor: boolean, - /** Image polaroid camera effect */ - polaroid: boolean, - /** Range: 0 (no pixelate) to N (number of pixels to pixelate) */ - pixelate: number, - }, + filter: Partial, // type definition end - /** Controlls gesture detection */ - gesture: { - enabled: boolean, - }, + gesture: Partial; - /** Controlls and configures all face-specific options: - * - face detection, face mesh detection, age, gender, emotion detection and face description - * Parameters: - * - enabled: true/false - * - modelPath: path for each of face models - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance - * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles - * - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor - */ - face: { - enabled: boolean, - detector: { - modelPath: string, - rotation: boolean, - maxDetected: number, - skipFrames: number, - minConfidence: number, - iouThreshold: number, - return: boolean, - }, - mesh: { - enabled: boolean, - modelPath: string, - }, - iris: { - enabled: boolean, - modelPath: string, - }, - description: { - enabled: boolean, - modelPath: string, - skipFrames: number, - minConfidence: number, - }, - emotion: { - enabled: boolean, - minConfidence: number, - skipFrames: number, - modelPath: string, - }, - }, + face: Partial, - /** Controlls and configures all body detection specific options - * - enabled: true/false - * - modelPath: body pose model, can be absolute path or relative to modelBasePath - * - minConfidence: threshold for discarding a prediction - * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance - */ - body: { - enabled: boolean, - modelPath: string, - maxDetected: number, - minConfidence: number, - skipFrames: number, - }, + body: Partial, - /** Controlls and configures all hand detection specific options - * - enabled: true/false - * - landmarks: detect hand landmarks or just hand boundary box - * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance - * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted - */ - hand: { - enabled: boolean, - rotation: boolean, - skipFrames: number, - minConfidence: number, - iouThreshold: number, - maxDetected: number, - landmarks: boolean, - detector: { - modelPath: string, - }, - skeleton: { - modelPath: string, - }, - }, + hand: Partial, - /** Controlls and configures all object detection specific options - * - enabled: true/false - * - modelPath: object detection model, can be absolute path or relative to modelBasePath - * - minConfidence: minimum score that detection must have to return as valid object - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of detections to return - */ - object: { - enabled: boolean, - modelPath: string, - minConfidence: number, - iouThreshold: number, - maxDetected: number, - skipFrames: number, - }, + object: Partial, - /** Controlls and configures all body segmentation module - * removes background from input containing person - * if segmentation is enabled it will run as preprocessing task before any other model - * alternatively leave it disabled and use it on-demand using human.segmentation method which can - * remove background or replace it with user-provided background - * - * - enabled: true/false - * - modelPath: object detection model, can be absolute path or relative to modelBasePath - */ - segmentation: { - enabled: boolean, - modelPath: string, - }, + segmentation: Partial, } const config: Config = { diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 6d4e13d2..572fa47f 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -4,7 +4,7 @@ import { TRI468 as triangulation } from '../blazeface/coords'; import { mergeDeep, now } from '../helpers'; -import type { Result, Face, Body, Hand, Item, Gesture, Person } from '../result'; +import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult } from '../result'; /** * Draw Options @@ -139,7 +139,7 @@ function curves(ctx, points: [number, number, number?][] = [], localOptions) { } } -export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); @@ -164,7 +164,7 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function face(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); @@ -266,7 +266,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, dra } } -export async function body(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function body(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); @@ -376,7 +376,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, dra } } -export async function hand(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function hand(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); @@ -441,7 +441,7 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array, dra } } -export async function object(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function object(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); @@ -466,7 +466,7 @@ export async function object(inCanvas: HTMLCanvasElement, result: Array, d } } -export async function person(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { +export async function person(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; const ctx = getCanvasContext(inCanvas); diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index 423a9920..28d29910 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -4,7 +4,7 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { Body } from '../result'; +import { BodyResult } from '../result'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; @@ -22,7 +22,7 @@ const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', export async function load(config: Config): Promise { if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)) as unknown as GraphModel; + model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath || '')) as unknown as GraphModel; if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath); else if (config.debug) log('load model:', model['modelUrl']); } else if (config.debug) log('cached model:', model['modelUrl']); @@ -46,8 +46,8 @@ function max2d(inputs, minScore) { }); } -export async function predict(image: Tensor, config: Config): Promise { - if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) { +export async function predict(image: Tensor, config: Config): Promise { + if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(keypoints).length > 0) { skipped++; return [{ id: 0, score, box, boxRaw, keypoints }]; } @@ -76,7 +76,7 @@ export async function predict(image: Tensor, config: Config): Promise { for (let id = 0; id < stack.length; id++) { // actual processing to get coordinates and score const [x, y, partScore] = max2d(stack[id], config.body.minConfidence); - if (score > config.body.minConfidence) { + if (score > (config.body?.minConfidence || 0)) { keypoints.push({ score: Math.round(100 * partScore) / 100, part: bodyParts[id], diff --git a/src/emotion/emotion.ts b/src/emotion/emotion.ts index 683fcd58..1fe56e73 100644 --- a/src/emotion/emotion.ts +++ b/src/emotion/emotion.ts @@ -19,8 +19,8 @@ const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when export async function load(config: Config): Promise { if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath)); - if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath); + model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion?.modelPath || '')); + if (!model || !model.modelUrl) log('load model failed:', config.face.emotion?.modelPath || ''); else if (config.debug) log('load model:', model.modelUrl); } else if (config.debug) log('cached model:', model.modelUrl); return model; @@ -28,7 +28,7 @@ export async function load(config: Config): Promise { export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; - if ((skipped < config.face.emotion.skipFrames) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) { + if ((skipped < (config.face.emotion?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) { skipped++; return last[idx]; } @@ -51,12 +51,12 @@ export async function predict(image: Tensor, config: Config, idx, count) { const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2)); tf.dispose(grayscale); const obj: Array<{ score: number, emotion: string }> = []; - if (config.face.emotion.enabled) { + if (config.face.emotion?.enabled) { const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation const data = await emotionT.data(); tf.dispose(emotionT); for (let i = 0; i < data.length; i++) { - if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); + if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); } obj.sort((a, b) => b.score - a.score); } diff --git a/src/face.ts b/src/face.ts index 32c321a5..9ec100b7 100644 --- a/src/face.ts +++ b/src/face.ts @@ -8,7 +8,7 @@ import * as tf from '../dist/tfjs.esm.js'; import * as facemesh from './blazeface/facemesh'; import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; -import { Face } from './result'; +import { FaceResult } from './result'; import { Tensor } from './tfjs/types'; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars @@ -145,7 +145,7 @@ const calculateFaceAngle = (face, imageSize): { return { angle, matrix, gaze }; }; -export const detectFace = async (parent /* instance of human */, input: Tensor): Promise => { +export const detectFace = async (parent /* instance of human */, input: Tensor): Promise => { // run facemesh, includes blazeface and iris // eslint-disable-next-line no-async-promise-executor let timeStamp; @@ -155,7 +155,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): let emotionRes; let embeddingRes; let descRes; - const faceRes: Array = []; + const faceRes: Array = []; parent.state = 'run:face'; timeStamp = now(); const faces = await facemesh.predict(input, parent.config); diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 6c317ed4..9bbd773f 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -23,10 +23,10 @@ let skipped = Number.MAX_SAFE_INTEGER; type DB = Array<{ name: string, source: string, embedding: number[] }>; export async function load(config: Config): Promise { - const modelUrl = join(config.modelBasePath, config.face.description.modelPath); + const modelUrl = join(config.modelBasePath, config.face.description?.modelPath || ''); if (!model) { model = await tf.loadGraphModel(modelUrl) as unknown as GraphModel; - if (!model) log('load model failed:', config.face.description.modelPath); + if (!model) log('load model failed:', config.face.description?.modelPath || ''); else if (config.debug) log('load model:', modelUrl); } else if (config.debug) log('cached model:', modelUrl); return model; @@ -112,7 +112,7 @@ export function enhance(input): Tensor { export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; - if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { + if ((skipped < (config.face.description?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { skipped++; return last[idx]; } @@ -128,13 +128,13 @@ export async function predict(image: Tensor, config: Config, idx, count) { descriptor: [], }; - if (config.face.description.enabled) resT = await model.predict(enhanced); + if (config.face.description?.enabled) resT = await model.predict(enhanced); tf.dispose(enhanced); if (resT) { const gender = await resT.find((t) => t.shape[1] === 1).data(); const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; - if (confidence > config.face.description.minConfidence) { + if (confidence > (config.face.description?.minConfidence || 0)) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.genderScore = Math.min(0.99, confidence); } diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index a5b4b25a..e4d727d6 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -2,7 +2,7 @@ * Gesture detection module */ -import { Gesture } from '../result'; +import { GestureResult } from '../result'; import * as fingerPose from '../fingerpose/fingerpose'; /** @@ -39,7 +39,7 @@ export type HandGesture = | 'victory' | 'thumbs up'; -export const body = (res): Gesture[] => { +export const body = (res): GestureResult[] => { if (!res) return []; const gestures: Array<{ body: number, gesture: BodyGesture }> = []; for (let i = 0; i < res.length; i++) { @@ -59,7 +59,7 @@ export const body = (res): Gesture[] => { return gestures; }; -export const face = (res): Gesture[] => { +export const face = (res): GestureResult[] => { if (!res) return []; const gestures: Array<{ face: number, gesture: FaceGesture }> = []; for (let i = 0; i < res.length; i++) { @@ -80,7 +80,7 @@ export const face = (res): Gesture[] => { return gestures; }; -export const iris = (res): Gesture[] => { +export const iris = (res): GestureResult[] => { if (!res) return []; const gestures: Array<{ iris: number, gesture: IrisGesture }> = []; for (let i = 0; i < res.length; i++) { @@ -118,7 +118,7 @@ export const iris = (res): Gesture[] => { return gestures; }; -export const hand = (res): Gesture[] => { +export const hand = (res): GestureResult[] => { if (!res) return []; const gestures: Array<{ hand: number, gesture: HandGesture }> = []; for (let i = 0; i < res.length; i++) { diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index d18db5e8..0ecd6f1b 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -7,7 +7,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; import * as handpipeline from './handpipeline'; import * as fingerPose from '../fingerpose/fingerpose'; -import { Hand } from '../result'; +import { HandResult } from '../result'; import { Tensor, GraphModel } from '../tfjs/types'; import { Config } from '../config'; @@ -24,10 +24,10 @@ let handDetectorModel: GraphModel | null; let handPoseModel: GraphModel | null; let handPipeline: handpipeline.HandPipeline; -export async function predict(input: Tensor, config: Config): Promise { +export async function predict(input: Tensor, config: Config): Promise { const predictions = await handPipeline.estimateHands(input, config); if (!predictions) return []; - const hands: Array = []; + const hands: Array = []; for (let i = 0; i < predictions.length; i++) { const annotations = {}; if (predictions[i].landmarks) { @@ -72,8 +72,8 @@ export async function predict(input: Tensor, config: Config): Promise { box, boxRaw, keypoints, - annotations: annotations as Hand['annotations'], - landmarks: landmarks as Hand['landmarks'], + annotations: annotations as HandResult['annotations'], + landmarks: landmarks as HandResult['landmarks'], }); } return hands; @@ -82,13 +82,13 @@ export async function predict(input: Tensor, config: Config): Promise { export async function load(config: Config): Promise<[GraphModel | null, GraphModel | null]> { if (!handDetectorModel || !handPoseModel) { [handDetectorModel, handPoseModel] = await Promise.all([ - config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) as unknown as GraphModel : null, - config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) as unknown as GraphModel : null, + config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector?.modelPath || ''), { fromTFHub: (config.hand.detector?.modelPath || '').includes('tfhub.dev') }) as unknown as GraphModel : null, + config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton?.modelPath || ''), { fromTFHub: (config.hand.skeleton?.modelPath || '').includes('tfhub.dev') }) as unknown as GraphModel : null, ]); if (config.hand.enabled) { - if (!handDetectorModel || !handDetectorModel['modelUrl']) log('load model failed:', config.hand.detector.modelPath); + if (!handDetectorModel || !handDetectorModel['modelUrl']) log('load model failed:', config.hand.detector?.modelPath || ''); else if (config.debug) log('load model:', handDetectorModel['modelUrl']); - if (!handPoseModel || !handPoseModel['modelUrl']) log('load model failed:', config.hand.skeleton.modelPath); + if (!handPoseModel || !handPoseModel['modelUrl']) log('load model failed:', config.hand.skeleton?.modelPath || ''); else if (config.debug) log('load model:', handPoseModel['modelUrl']); } } else { diff --git a/src/human.ts b/src/human.ts index eabe89dd..0c818e3a 100644 --- a/src/human.ts +++ b/src/human.ts @@ -4,7 +4,7 @@ import { log, now, mergeDeep } from './helpers'; import { Config, defaults } from './config'; -import { Result, Face, Hand, Body, Item, Gesture } from './result'; +import { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult } from './result'; import * as sysinfo from './sysinfo'; import * as tf from '../dist/tfjs.esm.js'; import * as backend from './tfjs/backend'; @@ -30,8 +30,9 @@ import * as app from '../package.json'; import { Tensor, GraphModel } from './tfjs/types'; // export types -export { Config } from './config'; -export type { Result, Face, Hand, Body, Item, Gesture, Person } from './result'; +export * from './config'; +export * from './result'; + export type { DrawOptions } from './draw/draw'; /** Defines all possible input types for **Human** detection @@ -101,16 +102,6 @@ export class Human { canvas: typeof draw.canvas, all: typeof draw.all, }; - /** Types used by Human */ - static Config: Config; - static Result: Result; - static Face: Face; - static Hand: Hand; - static Body: Body; - static Item: Item; - static Gesture: Gesture; - static Person: Gesture - static DrawOptions: draw.DrawOptions; /** @internal: Currently loaded models */ models: { face: [unknown, GraphModel | null, GraphModel | null] | null, @@ -529,10 +520,10 @@ export class Human { // prepare where to store model results // keep them with weak typing as it can be promise or not - let faceRes: Face[] | Promise | never[] = []; - let bodyRes: Body[] | Promise | never[] = []; - let handRes: Hand[] | Promise | never[] = []; - let objectRes: Item[] | Promise | never[] = []; + let faceRes: FaceResult[] | Promise | never[] = []; + let bodyRes: BodyResult[] | Promise | never[] = []; + let handRes: HandResult[] | Promise | never[] = []; + let objectRes: ObjectResult[] | Promise | never[] = []; // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion if (this.config.async) { @@ -549,18 +540,18 @@ export class Human { // run body: can be posenet, blazepose, efficientpose, movenet this.analyze('Start Body:'); if (this.config.async) { - if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : []; if (this.performance.body) delete this.performance.body; } else { this.state = 'run:body'; timeStamp = now(); - if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : []; - else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.body = elapsedTime; } @@ -583,14 +574,14 @@ export class Human { // run nanodet this.analyze('Start Object:'); if (this.config.async) { - if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : []; - else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : []; + if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : []; + else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : []; if (this.performance.object) delete this.performance.object; } else { this.state = 'run:object'; timeStamp = now(); - if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : []; - else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : []; + if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : []; + else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.object = elapsedTime; } @@ -600,7 +591,7 @@ export class Human { if (this.config.async) [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]); // run gesture analysis last - let gestureRes: Gesture[] = []; + let gestureRes: GestureResult[] = []; if (this.config.gesture.enabled) { timeStamp = now(); gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)]; @@ -612,15 +603,15 @@ export class Human { this.state = 'idle'; const shape = this.process?.tensor?.shape || []; this.result = { - face: faceRes as Face[], - body: bodyRes as Body[], - hand: handRes as Hand[], + face: faceRes as FaceResult[], + body: bodyRes as BodyResult[], + hand: handRes as HandResult[], gesture: gestureRes, - object: objectRes as Item[], + object: objectRes as ObjectResult[], performance: this.performance, canvas: this.process.canvas, timestamp: Date.now(), - get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); }, + get persons() { return persons.join(faceRes as FaceResult[], bodyRes as BodyResult[], handRes as HandResult[], gestureRes, shape); }, }; // finally dispose input tensor diff --git a/src/image/image.ts b/src/image/image.ts index 5db37a2a..5df88412 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -57,10 +57,10 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, } // create our canvas and resize it if needed - if (config.filter.width > 0) targetWidth = config.filter.width; - else if (config.filter.height > 0) targetWidth = originalWidth * (config.filter.height / originalHeight); - if (config.filter.height > 0) targetHeight = config.filter.height; - else if (config.filter.width > 0) targetHeight = originalHeight * (config.filter.width / originalWidth); + if ((config.filter.width || 0) > 0) targetWidth = config.filter.width; + else if ((config.filter.height || 0) > 0) targetWidth = originalWidth * ((config.filter.height || 0) / originalHeight); + if ((config.filter.height || 0) > 0) targetHeight = config.filter.height; + else if ((config.filter.width || 0) > 0) targetHeight = originalHeight * ((config.filter.width || 0) / originalWidth); if (!targetWidth || !targetHeight) throw new Error('Human: Input cannot determine dimension'); if (!inCanvas || (inCanvas?.width !== targetWidth) || (inCanvas?.height !== targetHeight)) { inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); diff --git a/src/interpolate.ts b/src/interpolate.ts index 835fa80f..54c28d46 100644 --- a/src/interpolate.ts +++ b/src/interpolate.ts @@ -2,7 +2,7 @@ * Module that interpolates results for smoother animations */ -import type { Result, Face, Body, Hand, Item, Gesture, Person } from './result'; +import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult } from './result'; const bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 }; @@ -26,7 +26,7 @@ export function calc(newResult: Result): Result { // interpolate body results if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) { - bufferedResult.body = JSON.parse(JSON.stringify(newResult.body as Body[])); // deep clone once + bufferedResult.body = JSON.parse(JSON.stringify(newResult.body as BodyResult[])); // deep clone once } else { for (let i = 0; i < newResult.body.length; i++) { const box = newResult.body[i].box // update box @@ -52,7 +52,7 @@ export function calc(newResult: Result): Result { // interpolate hand results if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) { - bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand as Hand[])); // deep clone once + bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand as HandResult[])); // deep clone once } else { for (let i = 0; i < newResult.hand.length; i++) { const box = (newResult.hand[i].box// update box @@ -69,13 +69,13 @@ export function calc(newResult: Result): Result { annotations[key] = newResult.hand[i].annotations[key] .map((val, j) => val.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor)); } - bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as Hand['annotations'] }; // shallow clone plus updated values + bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as HandResult['annotations'] }; // shallow clone plus updated values } } // interpolate face results if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) { - bufferedResult.face = JSON.parse(JSON.stringify(newResult.face as Face[])); // deep clone once + bufferedResult.face = JSON.parse(JSON.stringify(newResult.face as FaceResult[])); // deep clone once } else { for (let i = 0; i < newResult.face.length; i++) { const box = (newResult.face[i].box // update box @@ -104,7 +104,7 @@ export function calc(newResult: Result): Result { // interpolate object detection results if (!bufferedResult.object || (newResult.object.length !== bufferedResult.object.length)) { - bufferedResult.object = JSON.parse(JSON.stringify(newResult.object as Item[])); // deep clone once + bufferedResult.object = JSON.parse(JSON.stringify(newResult.object as ObjectResult[])); // deep clone once } else { for (let i = 0; i < newResult.object.length; i++) { const box = (newResult.object[i].box // update box @@ -119,7 +119,7 @@ export function calc(newResult: Result): Result { if (newResult.persons) { const newPersons = newResult.persons; // trigger getter function if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) { - bufferedResult.persons = JSON.parse(JSON.stringify(newPersons as Person[])); + bufferedResult.persons = JSON.parse(JSON.stringify(newPersons as PersonResult[])); } else { for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow bufferedResult.persons[i].box = (newPersons[i].box @@ -129,7 +129,7 @@ export function calc(newResult: Result): Result { } // just copy latest gestures without interpolation - if (newResult.gesture) bufferedResult.gesture = newResult.gesture as Gesture[]; + if (newResult.gesture) bufferedResult.gesture = newResult.gesture as GestureResult[]; if (newResult.performance) bufferedResult.performance = newResult.performance; return bufferedResult; diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index 9fb4d957..d38b5f3c 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -4,7 +4,7 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; -import { Body } from '../result'; +import { BodyResult } from '../result'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; @@ -23,7 +23,7 @@ const bodyParts = ['nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftSh export async function load(config: Config): Promise { if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)) as unknown as GraphModel; + model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath || '')) as unknown as GraphModel; if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath); else if (config.debug) log('load model:', model['modelUrl']); } else if (config.debug) log('cached model:', model['modelUrl']); @@ -114,8 +114,8 @@ async function parseMultiPose(res, config, image) { return persons; } -export async function predict(image: Tensor, config: Config): Promise { - if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) { +export async function predict(image: Tensor, config: Config): Promise { + if ((skipped < (config.body.skipFrames || 0)) && config.skipFrame && Object.keys(keypoints).length > 0) { skipped++; return [{ id: 0, score, box, boxRaw, keypoints }]; } diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 99d3f69d..78b44785 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -5,17 +5,17 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; -import { Item } from '../result'; +import { ObjectResult } from '../result'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; let model; -let last: Item[] = []; +let last: ObjectResult[] = []; let skipped = Number.MAX_SAFE_INTEGER; export async function load(config: Config): Promise { if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath)); + model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath || '')); const inputs = Object.values(model.modelSignature['inputs']); model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null; if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`); @@ -27,7 +27,7 @@ export async function load(config: Config): Promise { async function process(res: Tensor, inputSize, outputShape, config: Config) { if (!res) return []; - const results: Array = []; + const results: Array = []; const detections = await res.array(); const squeezeT = tf.squeeze(res); tf.dispose(res); @@ -70,8 +70,8 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) { return results; } -export async function predict(input: Tensor, config: Config): Promise { - if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) { +export async function predict(input: Tensor, config: Config): Promise { + if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) { skipped++; return last; } diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 732153a6..8bdabf17 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -5,19 +5,19 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; -import { Item } from '../result'; +import { ObjectResult } from '../result'; import { GraphModel, Tensor } from '../tfjs/types'; import { Config } from '../config'; let model; -let last: Array = []; +let last: Array = []; let skipped = Number.MAX_SAFE_INTEGER; const scaleBox = 2.5; // increase box size export async function load(config: Config): Promise { if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath)); + model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath || '')); const inputs = Object.values(model.modelSignature['inputs']); model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null; if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`); @@ -29,7 +29,7 @@ export async function load(config: Config): Promise { async function process(res, inputSize, outputShape, config) { let id = 0; - let results: Array = []; + let results: Array = []; for (const strideSize of [1, 2, 4]) { // try each stride size as it detects large/medium/small objects // find scores, boxes, classes tf.tidy(async () => { // wrap in tidy to automatically deallocate temp tensors @@ -102,8 +102,8 @@ async function process(res, inputSize, outputShape, config) { return results; } -export async function predict(image: Tensor, config: Config): Promise { - if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) { +export async function predict(image: Tensor, config: Config): Promise { + if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) { skipped++; return last; } diff --git a/src/persons.ts b/src/persons.ts index 5d3f2671..1d45caf7 100644 --- a/src/persons.ts +++ b/src/persons.ts @@ -2,13 +2,13 @@ * Module that analyzes existing results and recombines them into a unified person object */ -import { Face, Body, Hand, Gesture, Person } from './result'; +import { FaceResult, BodyResult, HandResult, GestureResult, PersonResult } from './result'; -export function join(faces: Array, bodies: Array, hands: Array, gestures: Array, shape: Array | undefined): Array { +export function join(faces: Array, bodies: Array, hands: Array, gestures: Array, shape: Array | undefined): Array { let id = 0; - const persons: Array = []; + const persons: Array = []; for (const face of faces) { // person is defined primarily by face and then we append other objects as found - const person: Person = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] }; + const person: PersonResult = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] }; for (const body of bodies) { if (face.box[0] > body.box[0] // x within body && face.box[0] < body.box[0] + body.box[2] diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index 097c7a68..697f62a9 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -1,5 +1,5 @@ import * as kpt from './keypoints'; -import { Body } from '../result'; +import { BodyResult } from '../result'; export function eitherPointDoesntMeetConfidence(a: number, b: number, minConfidence: number) { return (a < minConfidence || b < minConfidence); @@ -30,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] { return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY]; } -export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array { +export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array { const scaleY = height / inputResolutionHeight; const scaleX = width / inputResolutionWidth; const scalePose = (pose, i) => ({ diff --git a/src/result.ts b/src/result.ts index ec7cf290..66f7027b 100644 --- a/src/result.ts +++ b/src/result.ts @@ -32,7 +32,7 @@ import { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/ge * - gaze: gaze direction as object with values for bearing in radians and relative strength * - tensor: face tensor as Tensor object which contains detected face */ -export interface Face { +export interface FaceResult { id: number score: number, boxScore: number, @@ -69,7 +69,7 @@ export interface Face { * - score: body part score value * - presence: body part presence value */ -export interface Body { +export interface BodyResult { id: number, score: number, box: [number, number, number, number], @@ -94,7 +94,7 @@ export interface Body { * - annotations: annotated landmarks for each hand part with keypoints * - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings */ -export interface Hand { +export interface HandResult { id: number, score: number, box: [number, number, number, number], @@ -122,7 +122,7 @@ export interface Hand { * - center: optional center point as array of [x, y], normalized to image resolution * - centerRaw: optional center point as array of [x, y], normalized to range 0..1 */ -export interface Item { +export interface ObjectResult { id: number, score: number, class: number, @@ -139,7 +139,7 @@ export interface Item { * - part: part name and number where gesture was detected: face, iris, body, hand * - gesture: gesture detected */ -export type Gesture = +export type GestureResult = { 'face': number, gesture: FaceGesture } | { 'iris': number, gesture: IrisGesture } | { 'body': number, gesture: BodyGesture } @@ -157,12 +157,12 @@ export type Gesture = * - box: bounding box: x, y, width, height normalized to input image resolution * - boxRaw: bounding box: x, y, width, height normalized to 0..1 */ -export interface Person { +export interface PersonResult { id: number, - face: Face, - body: Body | null, - hands: { left: Hand | null, right: Hand | null }, - gestures: Array, + face: FaceResult, + body: BodyResult | null, + hands: { left: HandResult | null, right: HandResult | null }, + gestures: Array, box: [number, number, number, number], boxRaw?: [number, number, number, number], } @@ -173,16 +173,16 @@ export interface Person { * Contains all possible detection results */ export interface Result { - /** {@link Face}: detection & analysis results */ - face: Array, - /** {@link Body}: detection & analysis results */ - body: Array, - /** {@link Hand}: detection & analysis results */ - hand: Array, - /** {@link Gesture}: detection & analysis results */ - gesture: Array, - /** {@link Object}: detection & analysis results */ - object: Array + /** {@link FaceResult}: detection & analysis results */ + face: Array, + /** {@link BodyResult}: detection & analysis results */ + body: Array, + /** {@link HandResult}: detection & analysis results */ + hand: Array, + /** {@link GestureResult}: detection & analysis results */ + gesture: Array, + /** {@link ItemResult}: detection & analysis results */ + object: Array /** global performance object with timing values for each operation */ performance: Record, /** optional processed canvas that can be used to draw input on screen */ @@ -190,5 +190,5 @@ export interface Result { /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ readonly timestamp: number, /** getter property that returns unified persons object */ - persons: Array, + persons: Array, }