diff --git a/.eslintrc.json b/.eslintrc.json index eab09b41..a4132990 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -39,6 +39,7 @@ "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/triple-slash-reference": "off", "@typescript-eslint/no-inferrable-types": "off", + "@typescript-eslint/no-empty-interface": ["error", { "allowSingleExtends": true }], "camelcase": "off", "class-methods-use-this": "off", "dot-notation": "off", diff --git a/CHANGELOG.md b/CHANGELOG.md index adcfa6de..24f99635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ## Changelog -### **HEAD -> main** 2021/10/23 mandic00@live.com +### **HEAD -> main** 2021/10/25 mandic00@live.com - time based caching - turn on minification diff --git a/TODO.md b/TODO.md index 4ed8aa55..b7a4086f 100644 --- a/TODO.md +++ b/TODO.md @@ -53,13 +53,16 @@ Object detection using CenterNet or NanoDet models is not working when using WAS ## Pending Release - Update to TFJS 3.10.0 +- Update to Node v17 for Dev environment - Time based caching - Multiple bug fixes - Utility class `human.env` - Add `skipTime` in addition to `skipFrames` - Updated configuration default values - Updated TS typings +- Updated JSDoc comments and **TypeDoc** documentation - Enhanced **MoveNet** post-processing +- Increase `human.similarity` resolution - Add optional **Anti-Spoof** module - Remove old **HandDetect** and **PoseNet** models from default installation - Refactor **ImageFX** module diff --git a/demo/index.js b/demo/index.js index 1875f21f..cdb4f751 100644 --- a/demo/index.js +++ b/demo/index.js @@ -1016,7 +1016,7 @@ async function main() { // create instance of human human = new Human(userConfig); - human.env.perfadd = true; + // human.env.perfadd = true; log('human version:', human.version); // we've merged human defaults with user config and now lets store it back so it can be accessed by methods such as menu diff --git a/package.json b/package.json index 8340086e..e92e4986 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "@tensorflow/tfjs-layers": "^3.10.0", "@tensorflow/tfjs-node": "^3.10.0", "@tensorflow/tfjs-node-gpu": "^3.10.0", - "@types/node": "^16.11.4", + "@types/node": "^16.11.5", "@typescript-eslint/eslint-plugin": "^5.1.0", "@typescript-eslint/parser": "^5.1.0", "@vladmandic/build": "^0.6.3", diff --git a/src/config.ts b/src/config.ts index 5f03b521..1c202181 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,57 +1,57 @@ /* eslint-disable indent */ /* eslint-disable no-multi-spaces */ +/** Generic config type inherited by all module types */ export interface GenericConfig { + /** @property is module enabled? */ enabled: boolean, + /** @property path to model json file */ modelPath: string, + /** @property how many max frames to go without re-running model if cached results are acceptable */ skipFrames: number, + /** @property how many max miliseconds to go without re-running model if cached results are acceptable */ skipTime: number, } /** Dectector part of face configuration */ export interface FaceDetectorConfig extends GenericConfig { + /** @property is face rotation correction performed after detecting face? */ rotation: boolean, + /** @property maximum number of detected faces */ maxDetected: number, + /** @property minimum confidence for a detected face before results are discarded */ minConfidence: number, + /** @property minimum overlap between two detected faces before one is discarded */ iouThreshold: number, + /** @property should face detection return face tensor to be used in some other extenrnal model? */ return: boolean, } /** Mesh part of face configuration */ -export type FaceMeshConfig = GenericConfig +export interface FaceMeshConfig extends GenericConfig {} /** Iris part of face configuration */ -export type FaceIrisConfig = GenericConfig +export interface FaceIrisConfig extends GenericConfig {} /** Description or face embedding part of face configuration * - also used by age and gender detection */ export interface FaceDescriptionConfig extends GenericConfig { + /** @property minimum confidence for a detected face before results are discarded */ minConfidence: number, } /** Emotion part of face configuration */ export interface FaceEmotionConfig extends GenericConfig { + /** @property minimum confidence for a detected face before results are discarded */ minConfidence: number, } -/** Emotion part of face configuration */ -export type FaceAntiSpoofConfig = GenericConfig +/** Anti-spoofing part of face configuration */ +export interface FaceAntiSpoofConfig extends GenericConfig {} -/** Controlls and configures all face-specific options: - * - face detection, face mesh detection, age, gender, emotion detection and face description - * - * Parameters: - * - enabled: true/false - * - modelPath: path for each of face models - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance - * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles - * - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor -*/ -export interface FaceConfig { - enabled: boolean, +/** Configures all face-specific options: face detection, mesh analysis, age, gender, emotion detection and face description */ +export interface FaceConfig extends GenericConfig { detector: Partial, mesh: Partial, iris: Partial, @@ -60,92 +60,58 @@ export interface FaceConfig { antispoof: Partial, } -/** Controlls and configures all body detection specific options - * - * Parameters: - * - enabled: true/false - * - modelPath: body pose model, can be absolute path or relative to modelBasePath - * - minConfidence: threshold for discarding a prediction - * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance - * - detector: optional body detector - * - * `maxDetected` is valid for `posenet` and `movenet-multipose` as other models are single-pose only - * `maxDetected` can be set to -1 to auto-detect based on number of detected faces - * - * Changing `modelPath` will change module responsible for hand detection and tracking - * Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json` -*/ +/** Configures all body detection specific options */ export interface BodyConfig extends GenericConfig { + /** @property maximum numboer of detected bodies */ maxDetected: number, + /** @property minimum confidence for a detected body before results are discarded */ minConfidence: number, detector?: { + /** @property path to optional body detector model json file */ modelPath: string }, } -/** Controls and configures all hand detection specific options - * - * Parameters: - * - enabled: true/false - * - landmarks: detect hand landmarks or just hand boundary box - * - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath - * - minConfidence: threshold for discarding a prediction - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance - * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted - * - * `maxDetected` can be set to -1 to auto-detect based on number of detected faces - * - * Changing `detector.modelPath` will change module responsible for hand detection and tracking - * Allowed values are `handdetect.json` and `handtrack.json` -*/ +/** Configures all hand detection specific options */ export interface HandConfig extends GenericConfig { + /** @property should hand rotation correction be performed after hand detection? */ rotation: boolean, + /** @property minimum confidence for a detected hand before results are discarded */ minConfidence: number, + /** @property minimum overlap between two detected hands before one is discarded */ iouThreshold: number, + /** @property maximum number of detected hands */ maxDetected: number, + /** @property should hand landmarks be detected or just return detected hand box */ landmarks: boolean, detector: { + /** @property path to hand detector model json */ modelPath?: string, }, skeleton: { + /** @property path to hand skeleton model json */ modelPath?: string, }, } -/** Controlls and configures all object detection specific options - * - enabled: true/false - * - modelPath: object detection model, can be absolute path or relative to modelBasePath - * - minConfidence: minimum score that detection must have to return as valid object - * - iouThreshold: ammount of overlap between two detected objects before one object is removed - * - maxDetected: maximum number of detections to return - * - * Changing `modelPath` will change module responsible for hand detection and tracking - * Allowed values are `mb3-centernet.json` and `nanodet.json` -*/ +/** Configures all object detection specific options */ export interface ObjectConfig extends GenericConfig { + /** @property minimum confidence for a detected objects before results are discarded */ minConfidence: number, + /** @property minimum overlap between two detected objects before one is discarded */ iouThreshold: number, + /** @property maximum number of detected objects */ maxDetected: number, } -/** Controlls and configures all body segmentation module +/** Configures all body segmentation module * removes background from input containing person * if segmentation is enabled it will run as preprocessing task before any other model * alternatively leave it disabled and use it on-demand using human.segmentation method which can * remove background or replace it with user-provided background - * - * - enabled: true/false - * - modelPath: object detection model, can be absolute path or relative to modelBasePath - * - blur: blur segmentation output by pixels for more realistic image - * - * Changing `modelPath` will change module responsible for hand detection and tracking - * Allowed values are `selfie.json` and `meet.json` - */ -export interface SegmentationConfig { - enabled: boolean, - modelPath: string, +export interface SegmentationConfig extends GenericConfig { + /** @property blur segmentation output by pixels for more realistic image */ blur: number, } @@ -154,6 +120,7 @@ export interface SegmentationConfig { * - image filters run with near-zero latency as they are executed on the GPU using WebGL */ export interface FilterConfig { + /** @property are image filters enabled? */ enabled: boolean, /** Resize input width * - if both width and height are set to 0, there is no resizing @@ -167,40 +134,41 @@ export interface FilterConfig { * - if both are set, values are used as-is */ height: number, - /** Return processed canvas imagedata in result */ + /** @property return processed canvas imagedata in result */ return: boolean, - /** Flip input as mirror image */ + /** @property flip input as mirror image */ flip: boolean, - /** Range: -1 (darken) to 1 (lighten) */ + /** @property range: -1 (darken) to 1 (lighten) */ brightness: number, - /** Range: -1 (reduce contrast) to 1 (increase contrast) */ + /** @property range: -1 (reduce contrast) to 1 (increase contrast) */ contrast: number, - /** Range: 0 (no sharpening) to 1 (maximum sharpening) */ + /** @property range: 0 (no sharpening) to 1 (maximum sharpening) */ sharpness: number, - /** Range: 0 (no blur) to N (blur radius in pixels) */ + /** @property range: 0 (no blur) to N (blur radius in pixels) */ blur: number - /** Range: -1 (reduce saturation) to 1 (increase saturation) */ + /** @property range: -1 (reduce saturation) to 1 (increase saturation) */ saturation: number, - /** Range: 0 (no change) to 360 (hue rotation in degrees) */ + /** @property range: 0 (no change) to 360 (hue rotation in degrees) */ hue: number, - /** Image negative */ + /** @property image negative */ negative: boolean, - /** Image sepia colors */ + /** @property image sepia colors */ sepia: boolean, - /** Image vintage colors */ + /** @property image vintage colors */ vintage: boolean, - /** Image kodachrome colors */ + /** @property image kodachrome colors */ kodachrome: boolean, - /** Image technicolor colors */ + /** @property image technicolor colors */ technicolor: boolean, - /** Image polaroid camera effect */ + /** @property image polaroid camera effect */ polaroid: boolean, - /** Range: 0 (no pixelate) to N (number of pixels to pixelate) */ + /** @property range: 0 (no pixelate) to N (number of pixels to pixelate) */ pixelate: number, } /** Controlls gesture detection */ export interface GestureConfig { + /** @property is gesture detection enabled? */ enabled: boolean, } @@ -257,11 +225,7 @@ export interface Config { /** Internal Variable */ skipAllowed: boolean; - /** Run input through image filters before inference - * - image filters run with near-zero latency as they are executed on the GPU - * - * {@link FilterConfig} - */ + /** {@link FilterConfig} */ filter: Partial, /** {@link GestureConfig} */ @@ -283,10 +247,7 @@ export interface Config { segmentation: Partial, } -/** - * [See all default Config values...](https://github.com/vladmandic/human/blob/main/src/config.ts#L244) - * - */ +/** - [See all default Config values...](https://github.com/vladmandic/human/blob/main/src/config.ts#L250) */ const config: Config = { backend: '', // select tfjs backend to use, leave empty to use default backend // for browser environments: 'webgl', 'wasm', 'cpu', or 'humangl' (which is a custom version of webgl) diff --git a/src/exports.ts b/src/exports.ts new file mode 100644 index 00000000..417e13ca --- /dev/null +++ b/src/exports.ts @@ -0,0 +1,36 @@ +import type { Tensor } from './tfjs/types'; +import type { env } from './util/env'; + +export * from './config'; +export * from './result'; + +export type { Tensor } from './tfjs/types'; +export type { DrawOptions } from './util/draw'; +export type { Descriptor } from './face/match'; +export type { Box, Point } from './result'; +export type { Models } from './models'; +export type { Env } from './util/env'; +export type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; +export { env } from './util/env'; + +/** Events dispatched by `human.events` + * - `create`: triggered when Human object is instantiated + * - `load`: triggered when models are loaded (explicitly or on-demand) + * - `image`: triggered when input image is processed + * - `result`: triggered when detection is complete + * - `warmup`: triggered when warmup is complete + */ +export type Events = 'create' | 'load' | 'image' | 'result' | 'warmup' | 'error'; + +/** Defines all possible canvas types */ +export type AnyCanvas = HTMLCanvasElement | OffscreenCanvas; +/** Defines all possible image types */ +export type AnyImage = HTMLImageElement | typeof Image +/** Defines all possible video types */ +export type AnyVideo = HTMLMediaElement | HTMLVideoElement +/** Defines all possible image objects */ +export type ImageObjects = ImageData | ImageBitmap +/** Defines possible externally defined canvas */ +export type ExternalCanvas = typeof env.Canvas; +/** Defines all possible input types for **Human** detection */ +export type Input = Tensor | AnyCanvas | AnyImage | AnyVideo | ImageObjects | ExternalCanvas; diff --git a/src/face/match.ts b/src/face/match.ts index ee9e7631..63692217 100644 --- a/src/face/match.ts +++ b/src/face/match.ts @@ -1,15 +1,13 @@ -/** Defines Descriptor type */ +/** Face descriptor type as number array */ export type Descriptor = Array /** Calculates distance between two descriptors - * - Minkowski distance algorithm of nth order if `order` is different than 2 - * - Euclidean distance if `order` is 2 (default) - * - * Options: - * - `order` - * - `multiplier` factor by how much to enhance difference analysis in range of 1..100 - * - * Note: No checks are performed for performance reasons so make sure to pass valid number arrays of equal length + * @param {object} options + * @param {number} options.order algorithm to use + * - Euclidean distance if `order` is 2 (default), Minkowski distance algorithm of nth order if `order` is higher than 2 + * @param {number} options.multiplier by how much to enhance difference analysis in range of 1..100 + * - default is 20 which normalizes results to similarity above 0.5 can be considered a match + * @returns {number} */ export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) { // general minkowski distance, euclidean distance is limited case where order is 2 @@ -21,7 +19,13 @@ export function distance(descriptor1: Descriptor, descriptor2: Descriptor, optio return (options.multiplier || 20) * sum; } -/** Calculates normalized similarity between two descriptors based on their `distance` +/** Calculates normalized similarity between two face descriptors based on their `distance` + * @param {object} options + * @param {number} options.order algorithm to use + * - Euclidean distance if `order` is 2 (default), Minkowski distance algorithm of nth order if `order` is higher than 2 + * @param {number} options.multiplier by how much to enhance difference analysis in range of 1..100 + * - default is 20 which normalizes results to similarity above 0.5 can be considered a match + * @returns {number} similarity between two face descriptors normalized to 0..1 range where 0 is no similarity and 1 is perfect similarity */ export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) { const dist = distance(descriptor1, descriptor2, options); @@ -33,13 +37,10 @@ export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, opt /** Matches given descriptor to a closest entry in array of descriptors * @param descriptor face descriptor * @param descriptors array of face descriptors to commpare given descriptor to - * - * Options: - * - `threshold` match will return result first result for which {@link distance} is below `threshold` even if there may be better results - * - `order` see {@link distance} method - * - `multiplier` see {@link distance} method - * - * @returns object with index, distance and similarity + * @param {object} options + * @param {number} options.order see {@link similarity} + * @param {number} options.multiplier see {@link similarity} + * @returns {object} * - `index` index array index where best match was found or -1 if no matches * - {@link distance} calculated `distance` of given descriptor to the best match * - {@link similarity} calculated normalized `similarity` of given descriptor to the best match diff --git a/src/human.ts b/src/human.ts index 736792f2..1338aa42 100644 --- a/src/human.ts +++ b/src/human.ts @@ -1,5 +1,6 @@ /** * Human main module + * @author */ // module imports @@ -30,56 +31,23 @@ import * as persons from './util/persons'; import * as posenet from './body/posenet'; import * as segmentation from './segmentation/segmentation'; import * as warmups from './warmup'; - // type definitions -import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result'; -import type { Tensor } from './tfjs/types'; -import type { DrawOptions } from './util/draw'; -import type { Input } from './image/image'; -import type { Config } from './config'; +import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './exports'; +// type exports +export * from './exports'; -/** Defines configuration options used by all **Human** methods */ -export * from './config'; - -/** Defines result types returned by all **Human** methods */ -export * from './result'; - -/** Defines DrawOptions used by `human.draw.*` methods */ -export type { DrawOptions } from './util/draw'; -export { env, Env } from './util/env'; - -/** Face descriptor type as number array */ -export type { Descriptor } from './face/match'; - -/** Box and Point primitives */ -export { Box, Point } from './result'; - -/** Defines all possible models used by **Human** library */ -export { Models } from './models'; - -/** Defines all possible input types for **Human** detection */ -export { Input } from './image/image'; - -/** Events dispatched by `human.events` - * - * - `create`: triggered when Human object is instantiated - * - `load`: triggered when models are loaded (explicitly or on-demand) - * - `image`: triggered when input image is processed - * - `result`: triggered when detection is complete - * - `warmup`: triggered when warmup is complete - */ -export type Events = 'create' | 'load' | 'image' | 'result' | 'warmup' | 'error'; - -/** Error message - * @typedef Error Type - */ -export type Error = { error: string }; - -/** Instance of TensorFlow/JS - * @external +/** Instance of TensorFlow/JS used by Human + * - Can be TFJS that is bundled with `Human` or a manually imported TFJS library + * @external [API](https://js.tensorflow.org/api/latest/) */ export type TensorFlow = typeof tf; +/** Error message */ +export type Error = { + /** @property error message */ + error: string, +}; + /** **Human** library main class * * All methods and properties are available only as members of Human class @@ -89,21 +57,19 @@ export type TensorFlow = typeof tf; * - Possible inputs: {@link Input} * * @param userConfig: {@link Config} - * @return instance + * @returns instance of {@link Human} */ export class Human { /** Current version of Human library in *semver* format */ version: string; /** Current configuration - * - Definition: {@link Config} - * - Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L292) + * - Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L250) */ config: Config; /** Last known result of detect run * - Can be accessed anytime after initial detection - * - Definition: {@link Result} */ result: Result; @@ -128,12 +94,7 @@ export class Human { env: Env; /** Draw helper classes that can draw detected objects on canvas using specified draw - * - options: {@link DrawOptions} global settings for all draw operations, can be overriden for each draw method - * - face: draw detected faces - * - body: draw detected people and body parts - * - hand: draw detected hands and hand parts - * - canvas: draw processed canvas which is a processed copy of the input - * - all: meta-function that performs: canvas, face, body, hand + * @property options global settings for all draw operations, can be overriden for each draw method {@link DrawOptions} */ draw: { canvas: typeof draw.canvas, face: typeof draw.face, body: typeof draw.body, hand: typeof draw.hand, gesture: typeof draw.gesture, object: typeof draw.object, person: typeof draw.person, all: typeof draw.all, options: DrawOptions }; @@ -144,7 +105,7 @@ export class Human { models: models.Models; /** Container for events dispatched by Human - * + * {@type} EventTarget * Possible events: * - `create`: triggered when Human object is instantiated * - `load`: triggered when models are loaded (explicitly or on-demand) @@ -169,9 +130,8 @@ export class Human { /** Constructor for **Human** library that is futher used for all operations * - * @param userConfig: {@link Config} - * - * @return instance: {@link Human} + * @param {Config} userConfig + * @returns {Human} */ constructor(userConfig?: Partial) { this.env = env; @@ -269,6 +229,7 @@ export class Human { /** Process input as return canvas and tensor * * @param input: {@link Input} + * @param {boolean} input.getTensor should image processing also return tensor or just canvas * @returns { tensor, canvas } */ image(input: Input, getTensor: boolean = true) { @@ -276,17 +237,17 @@ export class Human { } /** Segmentation method takes any input and returns processed canvas with body segmentation - * - Optional parameter background is used to fill the background with specific input * - Segmentation is not triggered as part of detect process * * Returns: - * - `data` as raw data array with per-pixel segmentation values - * - `canvas` as canvas which is input image filtered with segementation data and optionally merged with background image. canvas alpha values are set to segmentation values for easy merging - * - `alpha` as grayscale canvas that represents segmentation alpha values * * @param input: {@link Input} * @param background?: {@link Input} - * @returns { data, canvas, alpha } + * - Optional parameter background is used to fill the background with specific input + * @returns {object} + * - `data` as raw data array with per-pixel segmentation values + * - `canvas` as canvas which is input image filtered with segementation data and optionally merged with background image. canvas alpha values are set to segmentation values for easy merging + * - `alpha` as grayscale canvas that represents segmentation alpha values */ async segmentation(input: Input, background?: Input): Promise<{ data: number[], canvas: HTMLCanvasElement | OffscreenCanvas | null, alpha: HTMLCanvasElement | OffscreenCanvas | null }> { return segmentation.process(input, background, this.config); @@ -307,7 +268,7 @@ export class Human { * - Call to explictly register and initialize TFJS backend without any other operations * - Use when changing backend during runtime * - * @return Promise + * @returns {void} */ async init(): Promise { await backend.check(this, true); diff --git a/src/image/image.ts b/src/image/image.ts index 0ffe272a..921b4d20 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -4,14 +4,10 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as fxImage from './imagefx'; -import type { Tensor } from '../tfjs/types'; -import type { Config } from '../config'; +import type { Input, AnyCanvas, Tensor, Config } from '../exports'; import { env } from '../util/env'; import { log, now } from '../util/util'; -export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas; -export type AnyCanvas = HTMLCanvasElement | OffscreenCanvas; - const maxSize = 2048; // internal temp canvases let inCanvas: AnyCanvas | null = null; // use global variable to avoid recreating canvas on each frame diff --git a/src/result.ts b/src/result.ts index 26579a4b..1ed08bfb 100644 --- a/src/result.ts +++ b/src/result.ts @@ -5,148 +5,135 @@ import type { Tensor } from './tfjs/types'; import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; +/** generic box as [x, y, width, height] */ export type Box = [number, number, number, number]; +/** generic point as [x, y, z?] */ export type Point = [number, number, number?]; /** Face results - * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models - * Some values may be null if specific model is not enabled - * - * Each result has: - * - id: face id number - * - score: overal detection confidence score value - * - boxScore: face box detection confidence score value - * - faceScore: face keypoints detection confidence score value - * - box: face bounding box as array of [x, y, width, height], normalized to image resolution - * - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1 - * - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution - * - meshRaw: face keypoints as array of [x, y, z] points of face mesh, normalized to range 0..1 - * - annotations: annotated face keypoints as array of annotated face mesh points - * - age: age as value - * - gender: gender as value - * - genderScore: gender detection confidence score as value - * - emotion: emotions as array of possible emotions with their individual scores - * - embedding: facial descriptor as array of numerical elements - * - iris: iris distance from current viewpoint as distance value in centimeters for a typical camera - * field of view of 88 degrees. value should be adjusted manually as needed - * - real: anti-spoofing analysis to determine if face is real of fake - * - rotation: face rotiation that contains both angles and matrix used for 3d transformations - * - angle: face angle as object with values for roll, yaw and pitch angles - * - matrix: 3d transofrmation matrix as array of numeric values - * - gaze: gaze direction as object with values for bearing in radians and relative strength - * - tensor: face tensor as Tensor object which contains detected face + * - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models + * - Some values may be null if specific model is not enabled */ export interface FaceResult { + /** face id */ id: number + /** overall face score */ score: number, + /** detection score */ boxScore: number, + /** mesh score */ faceScore: number, + /** detected face box */ box: Box, + /** detected face box normalized to 0..1 */ boxRaw: Box, + /** detected face mesh */ mesh: Array + /** detected face mesh normalized to 0..1 */ meshRaw: Array + /** mesh keypoints combined into annotated results */ annotations: Record, + /** detected age */ age?: number, + /** detected gender */ gender?: string, + /** gender detection score */ genderScore?: number, + /** detected emotions */ emotion?: Array<{ score: number, emotion: string }>, + /** face descriptor */ embedding?: Array, + /** face iris distance from camera */ iris?: number, + /** face anti-spoofing result confidence */ real?: number, + /** face rotation details */ rotation?: { angle: { roll: number, yaw: number, pitch: number }, matrix: [number, number, number, number, number, number, number, number, number], gaze: { bearing: number, strength: number }, } + /** detected face as tensor that can be used in further pipelines */ tensor?: Tensor, } -export type BodyKeypoint = { +export interface BodyKeypoint { + /** body part name */ part: string, + /** body part position */ position: Point, + /** body part position normalized to 0..1 */ positionRaw: Point, + /** body part detection score */ score: number, } -/** Body results - * - * Each results has: - * - id: body id number - * - score: overall detection score - * - box: bounding box: x, y, width, height normalized to input image resolution - * - boxRaw: bounding box: x, y, width, height normalized to 0..1 - * - keypoints: array of keypoints - * - part: body part name - * - position: body part position with x,y,z coordinates - * - score: body part score value - * - presence: body part presence value - */ +/** Body results */ export interface BodyResult { + /** body id */ id: number, + /** body detection score */ score: number, + /** detected body box */ box: Box, + /** detected body box normalized to 0..1 */ boxRaw: Box, - annotations: Record>, + /** detected body keypoints */ keypoints: Array + /** detected body keypoints combined into annotated parts */ + annotations: Record>, } -/** Hand results - * - * Each result has: - * - id: hand id number - * - score: detection confidence score as value - * - box: bounding box: x, y, width, height normalized to input image resolution - * - boxRaw: bounding box: x, y, width, height normalized to 0..1 - * - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution - * - annotations: annotated landmarks for each hand part with keypoints - * - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings - */ +/** Hand results */ export interface HandResult { + /** hand id */ id: number, + /** hand overal score */ score: number, + /** hand detection score */ boxScore: number, + /** hand skelton score */ fingerScore: number, + /** detected hand box */ box: Box, + /** detected hand box normalized to 0..1 */ boxRaw: Box, + /** detected hand keypoints */ keypoints: Array, + /** detected hand class */ label: string, + /** detected hand keypoints combined into annotated parts */ annotations: Record< 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm', Array >, + /** detected hand parts annotated with part gestures */ landmarks: Record< 'index' | 'middle' | 'pinky' | 'ring' | 'thumb', { curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' } >, } -/** Object results -* -* Array of individual results with one object per detected gesture -* Each result has: -* - id: object id number -* - score as value -* - label as detected class name -* - box: bounding box: x, y, width, height normalized to input image resolution -* - boxRaw: bounding box: x, y, width, height normalized to 0..1 -* - center: optional center point as array of [x, y], normalized to image resolution -* - centerRaw: optional center point as array of [x, y], normalized to range 0..1 -*/ +/** Object results */ export interface ObjectResult { + /** object id */ id: number, + /** object detection score */ score: number, + /** detected object class id */ class: number, + /** detected object class name */ label: string, + /** detected object box */ box: Box, + /** detected object box normalized to 0..1 */ boxRaw: Box, } -/** Gesture results +/** Gesture combined results * @typedef Gesture Type - * - * Array of individual results with one object per detected gesture * Each result has: - * - part: part name and number where gesture was detected: face, iris, body, hand + * - part: part name and number where gesture was detected: `face`, `iris`, `body`, `hand` * - gesture: gesture detected */ export type GestureResult = @@ -156,24 +143,22 @@ export type GestureResult = | { 'hand': number, gesture: HandGesture } /** Person getter -* @interface Person Interface -* -* Each result has: -* - id: person id -* - face: face object -* - body: body object -* - hands: array of hand objects -* - gestures: array of gestures -* - box: bounding box: x, y, width, height normalized to input image resolution -* - boxRaw: bounding box: x, y, width, height normalized to 0..1 +* - Triggers combining all individual results into a virtual person object */ export interface PersonResult { + /** person id */ id: number, + /** face result that belongs to this person */ face: FaceResult, + /** body result that belongs to this person */ body: BodyResult | null, + /** left and right hand results that belong to this person */ hands: { left: HandResult | null, right: HandResult | null }, + /** detected gestures specific to this person */ gestures: Array, + /** box that defines the person */ box: Box, + /** box that defines the person normalized to 0..1 */ boxRaw?: Box, }