mirror of https://github.com/vladmandic/human
refactoring plus jsdoc comments
Vladimir Mandic
parent
2bd59f1276
commit
b395a74701
|
|
@ -39,6 +39,7 @@
|
|||
"@typescript-eslint/no-var-requires": "off",
|
||||
"@typescript-eslint/triple-slash-reference": "off",
|
||||
"@typescript-eslint/no-inferrable-types": "off",
|
||||
"@typescript-eslint/no-empty-interface": ["error", { "allowSingleExtends": true }],
|
||||
"camelcase": "off",
|
||||
"class-methods-use-this": "off",
|
||||
"dot-notation": "off",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/10/23 mandic00@live.com
|
||||
### **HEAD -> main** 2021/10/25 mandic00@live.com
|
||||
|
||||
- time based caching
|
||||
- turn on minification
|
||||
|
|
|
|||
3
TODO.md
3
TODO.md
|
|
@ -53,13 +53,16 @@ Object detection using CenterNet or NanoDet models is not working when using WAS
|
|||
## Pending Release
|
||||
|
||||
- Update to TFJS 3.10.0
|
||||
- Update to Node v17 for Dev environment
|
||||
- Time based caching
|
||||
- Multiple bug fixes
|
||||
- Utility class `human.env`
|
||||
- Add `skipTime` in addition to `skipFrames`
|
||||
- Updated configuration default values
|
||||
- Updated TS typings
|
||||
- Updated JSDoc comments and **TypeDoc** documentation
|
||||
- Enhanced **MoveNet** post-processing
|
||||
- Increase `human.similarity` resolution
|
||||
- Add optional **Anti-Spoof** module
|
||||
- Remove old **HandDetect** and **PoseNet** models from default installation
|
||||
- Refactor **ImageFX** module
|
||||
|
|
|
|||
|
|
@ -1016,7 +1016,7 @@ async function main() {
|
|||
|
||||
// create instance of human
|
||||
human = new Human(userConfig);
|
||||
human.env.perfadd = true;
|
||||
// human.env.perfadd = true;
|
||||
|
||||
log('human version:', human.version);
|
||||
// we've merged human defaults with user config and now lets store it back so it can be accessed by methods such as menu
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@
|
|||
"@tensorflow/tfjs-layers": "^3.10.0",
|
||||
"@tensorflow/tfjs-node": "^3.10.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.10.0",
|
||||
"@types/node": "^16.11.4",
|
||||
"@types/node": "^16.11.5",
|
||||
"@typescript-eslint/eslint-plugin": "^5.1.0",
|
||||
"@typescript-eslint/parser": "^5.1.0",
|
||||
"@vladmandic/build": "^0.6.3",
|
||||
|
|
|
|||
151
src/config.ts
151
src/config.ts
|
|
@ -1,57 +1,57 @@
|
|||
/* eslint-disable indent */
|
||||
/* eslint-disable no-multi-spaces */
|
||||
|
||||
/** Generic config type inherited by all module types */
|
||||
export interface GenericConfig {
|
||||
/** @property is module enabled? */
|
||||
enabled: boolean,
|
||||
/** @property path to model json file */
|
||||
modelPath: string,
|
||||
/** @property how many max frames to go without re-running model if cached results are acceptable */
|
||||
skipFrames: number,
|
||||
/** @property how many max miliseconds to go without re-running model if cached results are acceptable */
|
||||
skipTime: number,
|
||||
}
|
||||
|
||||
/** Dectector part of face configuration */
|
||||
export interface FaceDetectorConfig extends GenericConfig {
|
||||
/** @property is face rotation correction performed after detecting face? */
|
||||
rotation: boolean,
|
||||
/** @property maximum number of detected faces */
|
||||
maxDetected: number,
|
||||
/** @property minimum confidence for a detected face before results are discarded */
|
||||
minConfidence: number,
|
||||
/** @property minimum overlap between two detected faces before one is discarded */
|
||||
iouThreshold: number,
|
||||
/** @property should face detection return face tensor to be used in some other extenrnal model? */
|
||||
return: boolean,
|
||||
}
|
||||
|
||||
/** Mesh part of face configuration */
|
||||
export type FaceMeshConfig = GenericConfig
|
||||
export interface FaceMeshConfig extends GenericConfig {}
|
||||
|
||||
/** Iris part of face configuration */
|
||||
export type FaceIrisConfig = GenericConfig
|
||||
export interface FaceIrisConfig extends GenericConfig {}
|
||||
|
||||
/** Description or face embedding part of face configuration
|
||||
* - also used by age and gender detection
|
||||
*/
|
||||
export interface FaceDescriptionConfig extends GenericConfig {
|
||||
/** @property minimum confidence for a detected face before results are discarded */
|
||||
minConfidence: number,
|
||||
}
|
||||
|
||||
/** Emotion part of face configuration */
|
||||
export interface FaceEmotionConfig extends GenericConfig {
|
||||
/** @property minimum confidence for a detected face before results are discarded */
|
||||
minConfidence: number,
|
||||
}
|
||||
|
||||
/** Emotion part of face configuration */
|
||||
export type FaceAntiSpoofConfig = GenericConfig
|
||||
/** Anti-spoofing part of face configuration */
|
||||
export interface FaceAntiSpoofConfig extends GenericConfig {}
|
||||
|
||||
/** Controlls and configures all face-specific options:
|
||||
* - face detection, face mesh detection, age, gender, emotion detection and face description
|
||||
*
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - modelPath: path for each of face models
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
|
||||
* - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor
|
||||
*/
|
||||
export interface FaceConfig {
|
||||
enabled: boolean,
|
||||
/** Configures all face-specific options: face detection, mesh analysis, age, gender, emotion detection and face description */
|
||||
export interface FaceConfig extends GenericConfig {
|
||||
detector: Partial<FaceDetectorConfig>,
|
||||
mesh: Partial<FaceMeshConfig>,
|
||||
iris: Partial<FaceIrisConfig>,
|
||||
|
|
@ -60,92 +60,58 @@ export interface FaceConfig {
|
|||
antispoof: Partial<FaceAntiSpoofConfig>,
|
||||
}
|
||||
|
||||
/** Controlls and configures all body detection specific options
|
||||
*
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - modelPath: body pose model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
|
||||
* - detector: optional body detector
|
||||
*
|
||||
* `maxDetected` is valid for `posenet` and `movenet-multipose` as other models are single-pose only
|
||||
* `maxDetected` can be set to -1 to auto-detect based on number of detected faces
|
||||
*
|
||||
* Changing `modelPath` will change module responsible for hand detection and tracking
|
||||
* Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json`
|
||||
*/
|
||||
/** Configures all body detection specific options */
|
||||
export interface BodyConfig extends GenericConfig {
|
||||
/** @property maximum numboer of detected bodies */
|
||||
maxDetected: number,
|
||||
/** @property minimum confidence for a detected body before results are discarded */
|
||||
minConfidence: number,
|
||||
detector?: {
|
||||
/** @property path to optional body detector model json file */
|
||||
modelPath: string
|
||||
},
|
||||
}
|
||||
|
||||
/** Controls and configures all hand detection specific options
|
||||
*
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - landmarks: detect hand landmarks or just hand boundary box
|
||||
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
|
||||
*
|
||||
* `maxDetected` can be set to -1 to auto-detect based on number of detected faces
|
||||
*
|
||||
* Changing `detector.modelPath` will change module responsible for hand detection and tracking
|
||||
* Allowed values are `handdetect.json` and `handtrack.json`
|
||||
*/
|
||||
/** Configures all hand detection specific options */
|
||||
export interface HandConfig extends GenericConfig {
|
||||
/** @property should hand rotation correction be performed after hand detection? */
|
||||
rotation: boolean,
|
||||
/** @property minimum confidence for a detected hand before results are discarded */
|
||||
minConfidence: number,
|
||||
/** @property minimum overlap between two detected hands before one is discarded */
|
||||
iouThreshold: number,
|
||||
/** @property maximum number of detected hands */
|
||||
maxDetected: number,
|
||||
/** @property should hand landmarks be detected or just return detected hand box */
|
||||
landmarks: boolean,
|
||||
detector: {
|
||||
/** @property path to hand detector model json */
|
||||
modelPath?: string,
|
||||
},
|
||||
skeleton: {
|
||||
/** @property path to hand skeleton model json */
|
||||
modelPath?: string,
|
||||
},
|
||||
}
|
||||
|
||||
/** Controlls and configures all object detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: minimum score that detection must have to return as valid object
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of detections to return
|
||||
*
|
||||
* Changing `modelPath` will change module responsible for hand detection and tracking
|
||||
* Allowed values are `mb3-centernet.json` and `nanodet.json`
|
||||
*/
|
||||
/** Configures all object detection specific options */
|
||||
export interface ObjectConfig extends GenericConfig {
|
||||
/** @property minimum confidence for a detected objects before results are discarded */
|
||||
minConfidence: number,
|
||||
/** @property minimum overlap between two detected objects before one is discarded */
|
||||
iouThreshold: number,
|
||||
/** @property maximum number of detected objects */
|
||||
maxDetected: number,
|
||||
}
|
||||
|
||||
/** Controlls and configures all body segmentation module
|
||||
/** Configures all body segmentation module
|
||||
* removes background from input containing person
|
||||
* if segmentation is enabled it will run as preprocessing task before any other model
|
||||
* alternatively leave it disabled and use it on-demand using human.segmentation method which can
|
||||
* remove background or replace it with user-provided background
|
||||
*
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
* - blur: blur segmentation output by <number> pixels for more realistic image
|
||||
*
|
||||
* Changing `modelPath` will change module responsible for hand detection and tracking
|
||||
* Allowed values are `selfie.json` and `meet.json`
|
||||
|
||||
*/
|
||||
export interface SegmentationConfig {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
export interface SegmentationConfig extends GenericConfig {
|
||||
/** @property blur segmentation output by <number> pixels for more realistic image */
|
||||
blur: number,
|
||||
}
|
||||
|
||||
|
|
@ -154,6 +120,7 @@ export interface SegmentationConfig {
|
|||
* - image filters run with near-zero latency as they are executed on the GPU using WebGL
|
||||
*/
|
||||
export interface FilterConfig {
|
||||
/** @property are image filters enabled? */
|
||||
enabled: boolean,
|
||||
/** Resize input width
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
|
|
@ -167,40 +134,41 @@ export interface FilterConfig {
|
|||
* - if both are set, values are used as-is
|
||||
*/
|
||||
height: number,
|
||||
/** Return processed canvas imagedata in result */
|
||||
/** @property return processed canvas imagedata in result */
|
||||
return: boolean,
|
||||
/** Flip input as mirror image */
|
||||
/** @property flip input as mirror image */
|
||||
flip: boolean,
|
||||
/** Range: -1 (darken) to 1 (lighten) */
|
||||
/** @property range: -1 (darken) to 1 (lighten) */
|
||||
brightness: number,
|
||||
/** Range: -1 (reduce contrast) to 1 (increase contrast) */
|
||||
/** @property range: -1 (reduce contrast) to 1 (increase contrast) */
|
||||
contrast: number,
|
||||
/** Range: 0 (no sharpening) to 1 (maximum sharpening) */
|
||||
/** @property range: 0 (no sharpening) to 1 (maximum sharpening) */
|
||||
sharpness: number,
|
||||
/** Range: 0 (no blur) to N (blur radius in pixels) */
|
||||
/** @property range: 0 (no blur) to N (blur radius in pixels) */
|
||||
blur: number
|
||||
/** Range: -1 (reduce saturation) to 1 (increase saturation) */
|
||||
/** @property range: -1 (reduce saturation) to 1 (increase saturation) */
|
||||
saturation: number,
|
||||
/** Range: 0 (no change) to 360 (hue rotation in degrees) */
|
||||
/** @property range: 0 (no change) to 360 (hue rotation in degrees) */
|
||||
hue: number,
|
||||
/** Image negative */
|
||||
/** @property image negative */
|
||||
negative: boolean,
|
||||
/** Image sepia colors */
|
||||
/** @property image sepia colors */
|
||||
sepia: boolean,
|
||||
/** Image vintage colors */
|
||||
/** @property image vintage colors */
|
||||
vintage: boolean,
|
||||
/** Image kodachrome colors */
|
||||
/** @property image kodachrome colors */
|
||||
kodachrome: boolean,
|
||||
/** Image technicolor colors */
|
||||
/** @property image technicolor colors */
|
||||
technicolor: boolean,
|
||||
/** Image polaroid camera effect */
|
||||
/** @property image polaroid camera effect */
|
||||
polaroid: boolean,
|
||||
/** Range: 0 (no pixelate) to N (number of pixels to pixelate) */
|
||||
/** @property range: 0 (no pixelate) to N (number of pixels to pixelate) */
|
||||
pixelate: number,
|
||||
}
|
||||
|
||||
/** Controlls gesture detection */
|
||||
export interface GestureConfig {
|
||||
/** @property is gesture detection enabled? */
|
||||
enabled: boolean,
|
||||
}
|
||||
|
||||
|
|
@ -257,11 +225,7 @@ export interface Config {
|
|||
/** Internal Variable */
|
||||
skipAllowed: boolean;
|
||||
|
||||
/** Run input through image filters before inference
|
||||
* - image filters run with near-zero latency as they are executed on the GPU
|
||||
*
|
||||
* {@link FilterConfig}
|
||||
*/
|
||||
/** {@link FilterConfig} */
|
||||
filter: Partial<FilterConfig>,
|
||||
|
||||
/** {@link GestureConfig} */
|
||||
|
|
@ -283,10 +247,7 @@ export interface Config {
|
|||
segmentation: Partial<SegmentationConfig>,
|
||||
}
|
||||
|
||||
/**
|
||||
* [See all default Config values...](https://github.com/vladmandic/human/blob/main/src/config.ts#L244)
|
||||
*
|
||||
*/
|
||||
/** - [See all default Config values...](https://github.com/vladmandic/human/blob/main/src/config.ts#L250) */
|
||||
const config: Config = {
|
||||
backend: '', // select tfjs backend to use, leave empty to use default backend
|
||||
// for browser environments: 'webgl', 'wasm', 'cpu', or 'humangl' (which is a custom version of webgl)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
import type { Tensor } from './tfjs/types';
|
||||
import type { env } from './util/env';
|
||||
|
||||
export * from './config';
|
||||
export * from './result';
|
||||
|
||||
export type { Tensor } from './tfjs/types';
|
||||
export type { DrawOptions } from './util/draw';
|
||||
export type { Descriptor } from './face/match';
|
||||
export type { Box, Point } from './result';
|
||||
export type { Models } from './models';
|
||||
export type { Env } from './util/env';
|
||||
export type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
|
||||
export { env } from './util/env';
|
||||
|
||||
/** Events dispatched by `human.events`
|
||||
* - `create`: triggered when Human object is instantiated
|
||||
* - `load`: triggered when models are loaded (explicitly or on-demand)
|
||||
* - `image`: triggered when input image is processed
|
||||
* - `result`: triggered when detection is complete
|
||||
* - `warmup`: triggered when warmup is complete
|
||||
*/
|
||||
export type Events = 'create' | 'load' | 'image' | 'result' | 'warmup' | 'error';
|
||||
|
||||
/** Defines all possible canvas types */
|
||||
export type AnyCanvas = HTMLCanvasElement | OffscreenCanvas;
|
||||
/** Defines all possible image types */
|
||||
export type AnyImage = HTMLImageElement | typeof Image
|
||||
/** Defines all possible video types */
|
||||
export type AnyVideo = HTMLMediaElement | HTMLVideoElement
|
||||
/** Defines all possible image objects */
|
||||
export type ImageObjects = ImageData | ImageBitmap
|
||||
/** Defines possible externally defined canvas */
|
||||
export type ExternalCanvas = typeof env.Canvas;
|
||||
/** Defines all possible input types for **Human** detection */
|
||||
export type Input = Tensor | AnyCanvas | AnyImage | AnyVideo | ImageObjects | ExternalCanvas;
|
||||
|
|
@ -1,15 +1,13 @@
|
|||
/** Defines Descriptor type */
|
||||
/** Face descriptor type as number array */
|
||||
export type Descriptor = Array<number>
|
||||
|
||||
/** Calculates distance between two descriptors
|
||||
* - Minkowski distance algorithm of nth order if `order` is different than 2
|
||||
* - Euclidean distance if `order` is 2 (default)
|
||||
*
|
||||
* Options:
|
||||
* - `order`
|
||||
* - `multiplier` factor by how much to enhance difference analysis in range of 1..100
|
||||
*
|
||||
* Note: No checks are performed for performance reasons so make sure to pass valid number arrays of equal length
|
||||
* @param {object} options
|
||||
* @param {number} options.order algorithm to use
|
||||
* - Euclidean distance if `order` is 2 (default), Minkowski distance algorithm of nth order if `order` is higher than 2
|
||||
* @param {number} options.multiplier by how much to enhance difference analysis in range of 1..100
|
||||
* - default is 20 which normalizes results to similarity above 0.5 can be considered a match
|
||||
* @returns {number}
|
||||
*/
|
||||
export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) {
|
||||
// general minkowski distance, euclidean distance is limited case where order is 2
|
||||
|
|
@ -21,7 +19,13 @@ export function distance(descriptor1: Descriptor, descriptor2: Descriptor, optio
|
|||
return (options.multiplier || 20) * sum;
|
||||
}
|
||||
|
||||
/** Calculates normalized similarity between two descriptors based on their `distance`
|
||||
/** Calculates normalized similarity between two face descriptors based on their `distance`
|
||||
* @param {object} options
|
||||
* @param {number} options.order algorithm to use
|
||||
* - Euclidean distance if `order` is 2 (default), Minkowski distance algorithm of nth order if `order` is higher than 2
|
||||
* @param {number} options.multiplier by how much to enhance difference analysis in range of 1..100
|
||||
* - default is 20 which normalizes results to similarity above 0.5 can be considered a match
|
||||
* @returns {number} similarity between two face descriptors normalized to 0..1 range where 0 is no similarity and 1 is perfect similarity
|
||||
*/
|
||||
export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) {
|
||||
const dist = distance(descriptor1, descriptor2, options);
|
||||
|
|
@ -33,13 +37,10 @@ export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, opt
|
|||
/** Matches given descriptor to a closest entry in array of descriptors
|
||||
* @param descriptor face descriptor
|
||||
* @param descriptors array of face descriptors to commpare given descriptor to
|
||||
*
|
||||
* Options:
|
||||
* - `threshold` match will return result first result for which {@link distance} is below `threshold` even if there may be better results
|
||||
* - `order` see {@link distance} method
|
||||
* - `multiplier` see {@link distance} method
|
||||
*
|
||||
* @returns object with index, distance and similarity
|
||||
* @param {object} options
|
||||
* @param {number} options.order see {@link similarity}
|
||||
* @param {number} options.multiplier see {@link similarity}
|
||||
* @returns {object}
|
||||
* - `index` index array index where best match was found or -1 if no matches
|
||||
* - {@link distance} calculated `distance` of given descriptor to the best match
|
||||
* - {@link similarity} calculated normalized `similarity` of given descriptor to the best match
|
||||
|
|
|
|||
91
src/human.ts
91
src/human.ts
|
|
@ -1,5 +1,6 @@
|
|||
/**
|
||||
* Human main module
|
||||
* @author <https://github.com/vladmandic>
|
||||
*/
|
||||
|
||||
// module imports
|
||||
|
|
@ -30,56 +31,23 @@ import * as persons from './util/persons';
|
|||
import * as posenet from './body/posenet';
|
||||
import * as segmentation from './segmentation/segmentation';
|
||||
import * as warmups from './warmup';
|
||||
|
||||
// type definitions
|
||||
import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
|
||||
import type { Tensor } from './tfjs/types';
|
||||
import type { DrawOptions } from './util/draw';
|
||||
import type { Input } from './image/image';
|
||||
import type { Config } from './config';
|
||||
import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './exports';
|
||||
// type exports
|
||||
export * from './exports';
|
||||
|
||||
/** Defines configuration options used by all **Human** methods */
|
||||
export * from './config';
|
||||
|
||||
/** Defines result types returned by all **Human** methods */
|
||||
export * from './result';
|
||||
|
||||
/** Defines DrawOptions used by `human.draw.*` methods */
|
||||
export type { DrawOptions } from './util/draw';
|
||||
export { env, Env } from './util/env';
|
||||
|
||||
/** Face descriptor type as number array */
|
||||
export type { Descriptor } from './face/match';
|
||||
|
||||
/** Box and Point primitives */
|
||||
export { Box, Point } from './result';
|
||||
|
||||
/** Defines all possible models used by **Human** library */
|
||||
export { Models } from './models';
|
||||
|
||||
/** Defines all possible input types for **Human** detection */
|
||||
export { Input } from './image/image';
|
||||
|
||||
/** Events dispatched by `human.events`
|
||||
*
|
||||
* - `create`: triggered when Human object is instantiated
|
||||
* - `load`: triggered when models are loaded (explicitly or on-demand)
|
||||
* - `image`: triggered when input image is processed
|
||||
* - `result`: triggered when detection is complete
|
||||
* - `warmup`: triggered when warmup is complete
|
||||
*/
|
||||
export type Events = 'create' | 'load' | 'image' | 'result' | 'warmup' | 'error';
|
||||
|
||||
/** Error message
|
||||
* @typedef Error Type
|
||||
*/
|
||||
export type Error = { error: string };
|
||||
|
||||
/** Instance of TensorFlow/JS
|
||||
* @external
|
||||
/** Instance of TensorFlow/JS used by Human
|
||||
* - Can be TFJS that is bundled with `Human` or a manually imported TFJS library
|
||||
* @external [API](https://js.tensorflow.org/api/latest/)
|
||||
*/
|
||||
export type TensorFlow = typeof tf;
|
||||
|
||||
/** Error message */
|
||||
export type Error = {
|
||||
/** @property error message */
|
||||
error: string,
|
||||
};
|
||||
|
||||
/** **Human** library main class
|
||||
*
|
||||
* All methods and properties are available only as members of Human class
|
||||
|
|
@ -89,21 +57,19 @@ export type TensorFlow = typeof tf;
|
|||
* - Possible inputs: {@link Input}
|
||||
*
|
||||
* @param userConfig: {@link Config}
|
||||
* @return instance
|
||||
* @returns instance of {@link Human}
|
||||
*/
|
||||
export class Human {
|
||||
/** Current version of Human library in *semver* format */
|
||||
version: string;
|
||||
|
||||
/** Current configuration
|
||||
* - Definition: {@link Config}
|
||||
* - Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L292)
|
||||
* - Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L250)
|
||||
*/
|
||||
config: Config;
|
||||
|
||||
/** Last known result of detect run
|
||||
* - Can be accessed anytime after initial detection
|
||||
* - Definition: {@link Result}
|
||||
*/
|
||||
result: Result;
|
||||
|
||||
|
|
@ -128,12 +94,7 @@ export class Human {
|
|||
env: Env;
|
||||
|
||||
/** Draw helper classes that can draw detected objects on canvas using specified draw
|
||||
* - options: {@link DrawOptions} global settings for all draw operations, can be overriden for each draw method
|
||||
* - face: draw detected faces
|
||||
* - body: draw detected people and body parts
|
||||
* - hand: draw detected hands and hand parts
|
||||
* - canvas: draw processed canvas which is a processed copy of the input
|
||||
* - all: meta-function that performs: canvas, face, body, hand
|
||||
* @property options global settings for all draw operations, can be overriden for each draw method {@link DrawOptions}
|
||||
*/
|
||||
draw: { canvas: typeof draw.canvas, face: typeof draw.face, body: typeof draw.body, hand: typeof draw.hand, gesture: typeof draw.gesture, object: typeof draw.object, person: typeof draw.person, all: typeof draw.all, options: DrawOptions };
|
||||
|
||||
|
|
@ -144,7 +105,7 @@ export class Human {
|
|||
models: models.Models;
|
||||
|
||||
/** Container for events dispatched by Human
|
||||
*
|
||||
* {@type} EventTarget
|
||||
* Possible events:
|
||||
* - `create`: triggered when Human object is instantiated
|
||||
* - `load`: triggered when models are loaded (explicitly or on-demand)
|
||||
|
|
@ -169,9 +130,8 @@ export class Human {
|
|||
|
||||
/** Constructor for **Human** library that is futher used for all operations
|
||||
*
|
||||
* @param userConfig: {@link Config}
|
||||
*
|
||||
* @return instance: {@link Human}
|
||||
* @param {Config} userConfig
|
||||
* @returns {Human}
|
||||
*/
|
||||
constructor(userConfig?: Partial<Config>) {
|
||||
this.env = env;
|
||||
|
|
@ -269,6 +229,7 @@ export class Human {
|
|||
/** Process input as return canvas and tensor
|
||||
*
|
||||
* @param input: {@link Input}
|
||||
* @param {boolean} input.getTensor should image processing also return tensor or just canvas
|
||||
* @returns { tensor, canvas }
|
||||
*/
|
||||
image(input: Input, getTensor: boolean = true) {
|
||||
|
|
@ -276,17 +237,17 @@ export class Human {
|
|||
}
|
||||
|
||||
/** Segmentation method takes any input and returns processed canvas with body segmentation
|
||||
* - Optional parameter background is used to fill the background with specific input
|
||||
* - Segmentation is not triggered as part of detect process
|
||||
*
|
||||
* Returns:
|
||||
* - `data` as raw data array with per-pixel segmentation values
|
||||
* - `canvas` as canvas which is input image filtered with segementation data and optionally merged with background image. canvas alpha values are set to segmentation values for easy merging
|
||||
* - `alpha` as grayscale canvas that represents segmentation alpha values
|
||||
*
|
||||
* @param input: {@link Input}
|
||||
* @param background?: {@link Input}
|
||||
* @returns { data, canvas, alpha }
|
||||
* - Optional parameter background is used to fill the background with specific input
|
||||
* @returns {object}
|
||||
* - `data` as raw data array with per-pixel segmentation values
|
||||
* - `canvas` as canvas which is input image filtered with segementation data and optionally merged with background image. canvas alpha values are set to segmentation values for easy merging
|
||||
* - `alpha` as grayscale canvas that represents segmentation alpha values
|
||||
*/
|
||||
async segmentation(input: Input, background?: Input): Promise<{ data: number[], canvas: HTMLCanvasElement | OffscreenCanvas | null, alpha: HTMLCanvasElement | OffscreenCanvas | null }> {
|
||||
return segmentation.process(input, background, this.config);
|
||||
|
|
@ -307,7 +268,7 @@ export class Human {
|
|||
* - Call to explictly register and initialize TFJS backend without any other operations
|
||||
* - Use when changing backend during runtime
|
||||
*
|
||||
* @return Promise<void>
|
||||
* @returns {void}
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
await backend.check(this, true);
|
||||
|
|
|
|||
|
|
@ -4,14 +4,10 @@
|
|||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as fxImage from './imagefx';
|
||||
import type { Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import type { Input, AnyCanvas, Tensor, Config } from '../exports';
|
||||
import { env } from '../util/env';
|
||||
import { log, now } from '../util/util';
|
||||
|
||||
export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas;
|
||||
export type AnyCanvas = HTMLCanvasElement | OffscreenCanvas;
|
||||
|
||||
const maxSize = 2048;
|
||||
// internal temp canvases
|
||||
let inCanvas: AnyCanvas | null = null; // use global variable to avoid recreating canvas on each frame
|
||||
|
|
|
|||
141
src/result.ts
141
src/result.ts
|
|
@ -5,148 +5,135 @@
|
|||
import type { Tensor } from './tfjs/types';
|
||||
import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
|
||||
|
||||
/** generic box as [x, y, width, height] */
|
||||
export type Box = [number, number, number, number];
|
||||
/** generic point as [x, y, z?] */
|
||||
export type Point = [number, number, number?];
|
||||
|
||||
/** Face results
|
||||
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
|
||||
* Some values may be null if specific model is not enabled
|
||||
*
|
||||
* Each result has:
|
||||
* - id: face id number
|
||||
* - score: overal detection confidence score value
|
||||
* - boxScore: face box detection confidence score value
|
||||
* - faceScore: face keypoints detection confidence score value
|
||||
* - box: face bounding box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1
|
||||
* - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution
|
||||
* - meshRaw: face keypoints as array of [x, y, z] points of face mesh, normalized to range 0..1
|
||||
* - annotations: annotated face keypoints as array of annotated face mesh points
|
||||
* - age: age as value
|
||||
* - gender: gender as value
|
||||
* - genderScore: gender detection confidence score as value
|
||||
* - emotion: emotions as array of possible emotions with their individual scores
|
||||
* - embedding: facial descriptor as array of numerical elements
|
||||
* - iris: iris distance from current viewpoint as distance value in centimeters for a typical camera
|
||||
* field of view of 88 degrees. value should be adjusted manually as needed
|
||||
* - real: anti-spoofing analysis to determine if face is real of fake
|
||||
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations
|
||||
* - angle: face angle as object with values for roll, yaw and pitch angles
|
||||
* - matrix: 3d transofrmation matrix as array of numeric values
|
||||
* - gaze: gaze direction as object with values for bearing in radians and relative strength
|
||||
* - tensor: face tensor as Tensor object which contains detected face
|
||||
* - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
|
||||
* - Some values may be null if specific model is not enabled
|
||||
*/
|
||||
export interface FaceResult {
|
||||
/** face id */
|
||||
id: number
|
||||
/** overall face score */
|
||||
score: number,
|
||||
/** detection score */
|
||||
boxScore: number,
|
||||
/** mesh score */
|
||||
faceScore: number,
|
||||
/** detected face box */
|
||||
box: Box,
|
||||
/** detected face box normalized to 0..1 */
|
||||
boxRaw: Box,
|
||||
/** detected face mesh */
|
||||
mesh: Array<Point>
|
||||
/** detected face mesh normalized to 0..1 */
|
||||
meshRaw: Array<Point>
|
||||
/** mesh keypoints combined into annotated results */
|
||||
annotations: Record<string, Point[]>,
|
||||
/** detected age */
|
||||
age?: number,
|
||||
/** detected gender */
|
||||
gender?: string,
|
||||
/** gender detection score */
|
||||
genderScore?: number,
|
||||
/** detected emotions */
|
||||
emotion?: Array<{ score: number, emotion: string }>,
|
||||
/** face descriptor */
|
||||
embedding?: Array<number>,
|
||||
/** face iris distance from camera */
|
||||
iris?: number,
|
||||
/** face anti-spoofing result confidence */
|
||||
real?: number,
|
||||
/** face rotation details */
|
||||
rotation?: {
|
||||
angle: { roll: number, yaw: number, pitch: number },
|
||||
matrix: [number, number, number, number, number, number, number, number, number],
|
||||
gaze: { bearing: number, strength: number },
|
||||
}
|
||||
/** detected face as tensor that can be used in further pipelines */
|
||||
tensor?: Tensor,
|
||||
}
|
||||
|
||||
export type BodyKeypoint = {
|
||||
export interface BodyKeypoint {
|
||||
/** body part name */
|
||||
part: string,
|
||||
/** body part position */
|
||||
position: Point,
|
||||
/** body part position normalized to 0..1 */
|
||||
positionRaw: Point,
|
||||
/** body part detection score */
|
||||
score: number,
|
||||
}
|
||||
|
||||
/** Body results
|
||||
*
|
||||
* Each results has:
|
||||
* - id: body id number
|
||||
* - score: overall detection score
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - keypoints: array of keypoints
|
||||
* - part: body part name
|
||||
* - position: body part position with x,y,z coordinates
|
||||
* - score: body part score value
|
||||
* - presence: body part presence value
|
||||
*/
|
||||
/** Body results */
|
||||
export interface BodyResult {
|
||||
/** body id */
|
||||
id: number,
|
||||
/** body detection score */
|
||||
score: number,
|
||||
/** detected body box */
|
||||
box: Box,
|
||||
/** detected body box normalized to 0..1 */
|
||||
boxRaw: Box,
|
||||
annotations: Record<string, Array<Point[]>>,
|
||||
/** detected body keypoints */
|
||||
keypoints: Array<BodyKeypoint>
|
||||
/** detected body keypoints combined into annotated parts */
|
||||
annotations: Record<string, Array<Point[]>>,
|
||||
}
|
||||
|
||||
/** Hand results
|
||||
*
|
||||
* Each result has:
|
||||
* - id: hand id number
|
||||
* - score: detection confidence score as value
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part with keypoints
|
||||
* - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
|
||||
*/
|
||||
/** Hand results */
|
||||
export interface HandResult {
|
||||
/** hand id */
|
||||
id: number,
|
||||
/** hand overal score */
|
||||
score: number,
|
||||
/** hand detection score */
|
||||
boxScore: number,
|
||||
/** hand skelton score */
|
||||
fingerScore: number,
|
||||
/** detected hand box */
|
||||
box: Box,
|
||||
/** detected hand box normalized to 0..1 */
|
||||
boxRaw: Box,
|
||||
/** detected hand keypoints */
|
||||
keypoints: Array<Point>,
|
||||
/** detected hand class */
|
||||
label: string,
|
||||
/** detected hand keypoints combined into annotated parts */
|
||||
annotations: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
|
||||
Array<Point>
|
||||
>,
|
||||
/** detected hand parts annotated with part gestures */
|
||||
landmarks: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
|
||||
{ curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
|
||||
>,
|
||||
}
|
||||
|
||||
/** Object results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - id: object id number
|
||||
* - score as value
|
||||
* - label as detected class name
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - center: optional center point as array of [x, y], normalized to image resolution
|
||||
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
|
||||
*/
|
||||
/** Object results */
|
||||
export interface ObjectResult {
|
||||
/** object id */
|
||||
id: number,
|
||||
/** object detection score */
|
||||
score: number,
|
||||
/** detected object class id */
|
||||
class: number,
|
||||
/** detected object class name */
|
||||
label: string,
|
||||
/** detected object box */
|
||||
box: Box,
|
||||
/** detected object box normalized to 0..1 */
|
||||
boxRaw: Box,
|
||||
}
|
||||
|
||||
/** Gesture results
|
||||
/** Gesture combined results
|
||||
* @typedef Gesture Type
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - part: part name and number where gesture was detected: face, iris, body, hand
|
||||
* - part: part name and number where gesture was detected: `face`, `iris`, `body`, `hand`
|
||||
* - gesture: gesture detected
|
||||
*/
|
||||
export type GestureResult =
|
||||
|
|
@ -156,24 +143,22 @@ export type GestureResult =
|
|||
| { 'hand': number, gesture: HandGesture }
|
||||
|
||||
/** Person getter
|
||||
* @interface Person Interface
|
||||
*
|
||||
* Each result has:
|
||||
* - id: person id
|
||||
* - face: face object
|
||||
* - body: body object
|
||||
* - hands: array of hand objects
|
||||
* - gestures: array of gestures
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - Triggers combining all individual results into a virtual person object
|
||||
*/
|
||||
export interface PersonResult {
|
||||
/** person id */
|
||||
id: number,
|
||||
/** face result that belongs to this person */
|
||||
face: FaceResult,
|
||||
/** body result that belongs to this person */
|
||||
body: BodyResult | null,
|
||||
/** left and right hand results that belong to this person */
|
||||
hands: { left: HandResult | null, right: HandResult | null },
|
||||
/** detected gestures specific to this person */
|
||||
gestures: Array<GestureResult>,
|
||||
/** box that defines the person */
|
||||
box: Box,
|
||||
/** box that defines the person normalized to 0..1 */
|
||||
boxRaw?: Box,
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue