mirror of https://github.com/vladmandic/human
redefine config and result interfaces
parent
b6f3f6a494
commit
7c0751b15e
|
@ -11,6 +11,7 @@
|
|||
|
||||
### **HEAD -> main** 2021/09/11 mandic00@live.com
|
||||
|
||||
- start using partial definitions
|
||||
- implement event emitters
|
||||
- fix iife loader
|
||||
- simplify dependencies
|
||||
|
|
337
src/config.ts
337
src/config.ts
|
@ -8,9 +8,177 @@
|
|||
* @typedef Config
|
||||
*/
|
||||
|
||||
/** Controlls and configures all face-specific options:
|
||||
* - face detection, face mesh detection, age, gender, emotion detection and face description
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - modelPath: path for each of face models
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
|
||||
* - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor
|
||||
*/
|
||||
export interface FaceConfig {
|
||||
enabled: boolean,
|
||||
detector: {
|
||||
modelPath: string,
|
||||
rotation: boolean,
|
||||
maxDetected: number,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
return: boolean,
|
||||
},
|
||||
mesh: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
},
|
||||
iris: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
},
|
||||
description: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
},
|
||||
emotion: {
|
||||
enabled: boolean,
|
||||
minConfidence: number,
|
||||
skipFrames: number,
|
||||
modelPath: string,
|
||||
},
|
||||
}
|
||||
|
||||
/** Controlls and configures all body detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: body pose model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
|
||||
*/
|
||||
export interface BodyConfig {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
maxDetected: number,
|
||||
minConfidence: number,
|
||||
skipFrames: number,
|
||||
}
|
||||
|
||||
/** Controlls and configures all hand detection specific options
|
||||
* - enabled: true/false
|
||||
* - landmarks: detect hand landmarks or just hand boundary box
|
||||
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
|
||||
*/
|
||||
export interface HandConfig {
|
||||
enabled: boolean,
|
||||
rotation: boolean,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
maxDetected: number,
|
||||
landmarks: boolean,
|
||||
detector: {
|
||||
modelPath: string,
|
||||
},
|
||||
skeleton: {
|
||||
modelPath: string,
|
||||
},
|
||||
}
|
||||
|
||||
/** Controlls and configures all object detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: minimum score that detection must have to return as valid object
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of detections to return
|
||||
*/
|
||||
export interface ObjectConfig {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
maxDetected: number,
|
||||
skipFrames: number,
|
||||
}
|
||||
|
||||
/** Controlls and configures all body segmentation module
|
||||
* removes background from input containing person
|
||||
* if segmentation is enabled it will run as preprocessing task before any other model
|
||||
* alternatively leave it disabled and use it on-demand using human.segmentation method which can
|
||||
* remove background or replace it with user-provided background
|
||||
*
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
*/
|
||||
export interface SegmentationConfig {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
}
|
||||
|
||||
/** Run input through image filters before inference
|
||||
* - image filters run with near-zero latency as they are executed on the GPU
|
||||
*/
|
||||
export interface FilterConfig {
|
||||
enabled: boolean,
|
||||
/** Resize input width
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
width: number,
|
||||
/** Resize input height
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
height: number,
|
||||
/** Return processed canvas imagedata in result */
|
||||
return: boolean,
|
||||
/** Flip input as mirror image */
|
||||
flip: boolean,
|
||||
/** Range: -1 (darken) to 1 (lighten) */
|
||||
brightness: number,
|
||||
/** Range: -1 (reduce contrast) to 1 (increase contrast) */
|
||||
contrast: number,
|
||||
/** Range: 0 (no sharpening) to 1 (maximum sharpening) */
|
||||
sharpness: number,
|
||||
/** Range: 0 (no blur) to N (blur radius in pixels) */
|
||||
blur: number
|
||||
/** Range: -1 (reduce saturation) to 1 (increase saturation) */
|
||||
saturation: number,
|
||||
/** Range: 0 (no change) to 360 (hue rotation in degrees) */
|
||||
hue: number,
|
||||
/** Image negative */
|
||||
negative: boolean,
|
||||
/** Image sepia colors */
|
||||
sepia: boolean,
|
||||
/** Image vintage colors */
|
||||
vintage: boolean,
|
||||
/** Image kodachrome colors */
|
||||
kodachrome: boolean,
|
||||
/** Image technicolor colors */
|
||||
technicolor: boolean,
|
||||
/** Image polaroid camera effect */
|
||||
polaroid: boolean,
|
||||
/** Range: 0 (no pixelate) to N (number of pixels to pixelate) */
|
||||
pixelate: number,
|
||||
}
|
||||
|
||||
/** Controlls gesture detection */
|
||||
export interface GestureConfig {
|
||||
enabled: boolean,
|
||||
}
|
||||
|
||||
export interface Config {
|
||||
/** Backend used for TFJS operations */
|
||||
backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu' | null | string,
|
||||
// backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu' | null,
|
||||
backend: string;
|
||||
|
||||
/** Path to *.wasm files if backend is set to `wasm` */
|
||||
wasmPath: string,
|
||||
|
@ -25,7 +193,8 @@ export interface Config {
|
|||
* - warmup pre-initializes all models for faster inference but can take significant time on startup
|
||||
* - only used for `webgl` and `humangl` backends
|
||||
*/
|
||||
warmup: 'none' | 'face' | 'full' | 'body' | string,
|
||||
// warmup: 'none' | 'face' | 'full' | 'body' | string,
|
||||
warmup: string;
|
||||
|
||||
/** Base model path (typically starting with file://, http:// or https://) for all models
|
||||
* - individual modelPath values are relative to this path
|
||||
|
@ -47,170 +216,20 @@ export interface Config {
|
|||
/** Run input through image filters before inference
|
||||
* - image filters run with near-zero latency as they are executed on the GPU
|
||||
*/
|
||||
filter: {
|
||||
enabled: boolean,
|
||||
/** Resize input width
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
width: number,
|
||||
/** Resize input height
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
height: number,
|
||||
/** Return processed canvas imagedata in result */
|
||||
return: boolean,
|
||||
/** Flip input as mirror image */
|
||||
flip: boolean,
|
||||
/** Range: -1 (darken) to 1 (lighten) */
|
||||
brightness: number,
|
||||
/** Range: -1 (reduce contrast) to 1 (increase contrast) */
|
||||
contrast: number,
|
||||
/** Range: 0 (no sharpening) to 1 (maximum sharpening) */
|
||||
sharpness: number,
|
||||
/** Range: 0 (no blur) to N (blur radius in pixels) */
|
||||
blur: number
|
||||
/** Range: -1 (reduce saturation) to 1 (increase saturation) */
|
||||
saturation: number,
|
||||
/** Range: 0 (no change) to 360 (hue rotation in degrees) */
|
||||
hue: number,
|
||||
/** Image negative */
|
||||
negative: boolean,
|
||||
/** Image sepia colors */
|
||||
sepia: boolean,
|
||||
/** Image vintage colors */
|
||||
vintage: boolean,
|
||||
/** Image kodachrome colors */
|
||||
kodachrome: boolean,
|
||||
/** Image technicolor colors */
|
||||
technicolor: boolean,
|
||||
/** Image polaroid camera effect */
|
||||
polaroid: boolean,
|
||||
/** Range: 0 (no pixelate) to N (number of pixels to pixelate) */
|
||||
pixelate: number,
|
||||
},
|
||||
filter: Partial<FilterConfig>,
|
||||
// type definition end
|
||||
|
||||
/** Controlls gesture detection */
|
||||
gesture: {
|
||||
enabled: boolean,
|
||||
},
|
||||
gesture: Partial<GestureConfig>;
|
||||
|
||||
/** Controlls and configures all face-specific options:
|
||||
* - face detection, face mesh detection, age, gender, emotion detection and face description
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - modelPath: path for each of face models
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
|
||||
* - return: return extracted face as tensor for futher user processing, in which case user is reponsible for manually disposing the tensor
|
||||
*/
|
||||
face: {
|
||||
enabled: boolean,
|
||||
detector: {
|
||||
modelPath: string,
|
||||
rotation: boolean,
|
||||
maxDetected: number,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
return: boolean,
|
||||
},
|
||||
mesh: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
},
|
||||
iris: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
},
|
||||
description: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
},
|
||||
emotion: {
|
||||
enabled: boolean,
|
||||
minConfidence: number,
|
||||
skipFrames: number,
|
||||
modelPath: string,
|
||||
},
|
||||
},
|
||||
face: Partial<FaceConfig>,
|
||||
|
||||
/** Controlls and configures all body detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: body pose model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance
|
||||
*/
|
||||
body: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
maxDetected: number,
|
||||
minConfidence: number,
|
||||
skipFrames: number,
|
||||
},
|
||||
body: Partial<BodyConfig>,
|
||||
|
||||
/** Controlls and configures all hand detection specific options
|
||||
* - enabled: true/false
|
||||
* - landmarks: detect hand landmarks or just hand boundary box
|
||||
* - modelPath: paths for hand detector and hand skeleton models, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance
|
||||
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
|
||||
*/
|
||||
hand: {
|
||||
enabled: boolean,
|
||||
rotation: boolean,
|
||||
skipFrames: number,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
maxDetected: number,
|
||||
landmarks: boolean,
|
||||
detector: {
|
||||
modelPath: string,
|
||||
},
|
||||
skeleton: {
|
||||
modelPath: string,
|
||||
},
|
||||
},
|
||||
hand: Partial<HandConfig>,
|
||||
|
||||
/** Controlls and configures all object detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
* - minConfidence: minimum score that detection must have to return as valid object
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxDetected: maximum number of detections to return
|
||||
*/
|
||||
object: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
minConfidence: number,
|
||||
iouThreshold: number,
|
||||
maxDetected: number,
|
||||
skipFrames: number,
|
||||
},
|
||||
object: Partial<ObjectConfig>,
|
||||
|
||||
/** Controlls and configures all body segmentation module
|
||||
* removes background from input containing person
|
||||
* if segmentation is enabled it will run as preprocessing task before any other model
|
||||
* alternatively leave it disabled and use it on-demand using human.segmentation method which can
|
||||
* remove background or replace it with user-provided background
|
||||
*
|
||||
* - enabled: true/false
|
||||
* - modelPath: object detection model, can be absolute path or relative to modelBasePath
|
||||
*/
|
||||
segmentation: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
},
|
||||
segmentation: Partial<SegmentationConfig>,
|
||||
}
|
||||
|
||||
const config: Config = {
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import { TRI468 as triangulation } from '../blazeface/coords';
|
||||
import { mergeDeep, now } from '../helpers';
|
||||
import type { Result, Face, Body, Hand, Item, Gesture, Person } from '../result';
|
||||
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult } from '../result';
|
||||
|
||||
/**
|
||||
* Draw Options
|
||||
|
@ -139,7 +139,7 @@ function curves(ctx, points: [number, number, number?][] = [], localOptions) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<Gesture>, drawOptions?: DrawOptions) {
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<GestureResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
@ -164,7 +164,7 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array<Gesture
|
|||
}
|
||||
}
|
||||
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, drawOptions?: DrawOptions) {
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<FaceResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
@ -266,7 +266,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
|
|||
}
|
||||
}
|
||||
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, drawOptions?: DrawOptions) {
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<BodyResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
@ -376,7 +376,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
|
|||
}
|
||||
}
|
||||
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, drawOptions?: DrawOptions) {
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<HandResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
@ -441,7 +441,7 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
}
|
||||
}
|
||||
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<Item>, drawOptions?: DrawOptions) {
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<ObjectResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
@ -466,7 +466,7 @@ export async function object(inCanvas: HTMLCanvasElement, result: Array<Item>, d
|
|||
}
|
||||
}
|
||||
|
||||
export async function person(inCanvas: HTMLCanvasElement, result: Array<Person>, drawOptions?: DrawOptions) {
|
||||
export async function person(inCanvas: HTMLCanvasElement, result: Array<PersonResult>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
const ctx = getCanvasContext(inCanvas);
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { Body } from '../result';
|
||||
import { BodyResult } from '../result';
|
||||
import { GraphModel, Tensor } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
|
@ -22,7 +22,7 @@ const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist',
|
|||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)) as unknown as GraphModel;
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath || '')) as unknown as GraphModel;
|
||||
if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath);
|
||||
else if (config.debug) log('load model:', model['modelUrl']);
|
||||
} else if (config.debug) log('cached model:', model['modelUrl']);
|
||||
|
@ -46,8 +46,8 @@ function max2d(inputs, minScore) {
|
|||
});
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config): Promise<Body[]> {
|
||||
if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
|
||||
if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
skipped++;
|
||||
return [{ id: 0, score, box, boxRaw, keypoints }];
|
||||
}
|
||||
|
@ -76,7 +76,7 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
|
|||
for (let id = 0; id < stack.length; id++) {
|
||||
// actual processing to get coordinates and score
|
||||
const [x, y, partScore] = max2d(stack[id], config.body.minConfidence);
|
||||
if (score > config.body.minConfidence) {
|
||||
if (score > (config.body?.minConfidence || 0)) {
|
||||
keypoints.push({
|
||||
score: Math.round(100 * partScore) / 100,
|
||||
part: bodyParts[id],
|
||||
|
|
|
@ -19,8 +19,8 @@ const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when
|
|||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion.modelPath));
|
||||
if (!model || !model.modelUrl) log('load model failed:', config.face.emotion.modelPath);
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.face.emotion?.modelPath || ''));
|
||||
if (!model || !model.modelUrl) log('load model failed:', config.face.emotion?.modelPath || '');
|
||||
else if (config.debug) log('load model:', model.modelUrl);
|
||||
} else if (config.debug) log('cached model:', model.modelUrl);
|
||||
return model;
|
||||
|
@ -28,7 +28,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
|
||||
export async function predict(image: Tensor, config: Config, idx, count) {
|
||||
if (!model) return null;
|
||||
if ((skipped < config.face.emotion.skipFrames) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
|
||||
if ((skipped < (config.face.emotion?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
|
||||
skipped++;
|
||||
return last[idx];
|
||||
}
|
||||
|
@ -51,12 +51,12 @@ export async function predict(image: Tensor, config: Config, idx, count) {
|
|||
const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
|
||||
tf.dispose(grayscale);
|
||||
const obj: Array<{ score: number, emotion: string }> = [];
|
||||
if (config.face.emotion.enabled) {
|
||||
if (config.face.emotion?.enabled) {
|
||||
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
|
||||
const data = await emotionT.data();
|
||||
tf.dispose(emotionT);
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
|
||||
}
|
||||
obj.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@ import * as tf from '../dist/tfjs.esm.js';
|
|||
import * as facemesh from './blazeface/facemesh';
|
||||
import * as emotion from './emotion/emotion';
|
||||
import * as faceres from './faceres/faceres';
|
||||
import { Face } from './result';
|
||||
import { FaceResult } from './result';
|
||||
import { Tensor } from './tfjs/types';
|
||||
|
||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||
|
@ -145,7 +145,7 @@ const calculateFaceAngle = (face, imageSize): {
|
|||
return { angle, matrix, gaze };
|
||||
};
|
||||
|
||||
export const detectFace = async (parent /* instance of human */, input: Tensor): Promise<Face[]> => {
|
||||
export const detectFace = async (parent /* instance of human */, input: Tensor): Promise<FaceResult[]> => {
|
||||
// run facemesh, includes blazeface and iris
|
||||
// eslint-disable-next-line no-async-promise-executor
|
||||
let timeStamp;
|
||||
|
@ -155,7 +155,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
let emotionRes;
|
||||
let embeddingRes;
|
||||
let descRes;
|
||||
const faceRes: Array<Face> = [];
|
||||
const faceRes: Array<FaceResult> = [];
|
||||
parent.state = 'run:face';
|
||||
timeStamp = now();
|
||||
const faces = await facemesh.predict(input, parent.config);
|
||||
|
|
|
@ -23,10 +23,10 @@ let skipped = Number.MAX_SAFE_INTEGER;
|
|||
type DB = Array<{ name: string, source: string, embedding: number[] }>;
|
||||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
const modelUrl = join(config.modelBasePath, config.face.description.modelPath);
|
||||
const modelUrl = join(config.modelBasePath, config.face.description?.modelPath || '');
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(modelUrl) as unknown as GraphModel;
|
||||
if (!model) log('load model failed:', config.face.description.modelPath);
|
||||
if (!model) log('load model failed:', config.face.description?.modelPath || '');
|
||||
else if (config.debug) log('load model:', modelUrl);
|
||||
} else if (config.debug) log('cached model:', modelUrl);
|
||||
return model;
|
||||
|
@ -112,7 +112,7 @@ export function enhance(input): Tensor {
|
|||
|
||||
export async function predict(image: Tensor, config: Config, idx, count) {
|
||||
if (!model) return null;
|
||||
if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
|
||||
if ((skipped < (config.face.description?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
|
||||
skipped++;
|
||||
return last[idx];
|
||||
}
|
||||
|
@ -128,13 +128,13 @@ export async function predict(image: Tensor, config: Config, idx, count) {
|
|||
descriptor: <number[]>[],
|
||||
};
|
||||
|
||||
if (config.face.description.enabled) resT = await model.predict(enhanced);
|
||||
if (config.face.description?.enabled) resT = await model.predict(enhanced);
|
||||
tf.dispose(enhanced);
|
||||
|
||||
if (resT) {
|
||||
const gender = await resT.find((t) => t.shape[1] === 1).data();
|
||||
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
|
||||
if (confidence > config.face.description.minConfidence) {
|
||||
if (confidence > (config.face.description?.minConfidence || 0)) {
|
||||
obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
|
||||
obj.genderScore = Math.min(0.99, confidence);
|
||||
}
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
* Gesture detection module
|
||||
*/
|
||||
|
||||
import { Gesture } from '../result';
|
||||
import { GestureResult } from '../result';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
|
||||
/**
|
||||
|
@ -39,7 +39,7 @@ export type HandGesture =
|
|||
| 'victory'
|
||||
| 'thumbs up';
|
||||
|
||||
export const body = (res): Gesture[] => {
|
||||
export const body = (res): GestureResult[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ body: number, gesture: BodyGesture }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
@ -59,7 +59,7 @@ export const body = (res): Gesture[] => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const face = (res): Gesture[] => {
|
||||
export const face = (res): GestureResult[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ face: number, gesture: FaceGesture }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
@ -80,7 +80,7 @@ export const face = (res): Gesture[] => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const iris = (res): Gesture[] => {
|
||||
export const iris = (res): GestureResult[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ iris: number, gesture: IrisGesture }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
@ -118,7 +118,7 @@ export const iris = (res): Gesture[] => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const hand = (res): Gesture[] => {
|
||||
export const hand = (res): GestureResult[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ hand: number, gesture: HandGesture }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
|
|
@ -7,7 +7,7 @@ import * as tf from '../../dist/tfjs.esm.js';
|
|||
import * as handdetector from './handdetector';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
import { Hand } from '../result';
|
||||
import { HandResult } from '../result';
|
||||
import { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
|
@ -24,10 +24,10 @@ let handDetectorModel: GraphModel | null;
|
|||
let handPoseModel: GraphModel | null;
|
||||
let handPipeline: handpipeline.HandPipeline;
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
|
||||
export async function predict(input: Tensor, config: Config): Promise<HandResult[]> {
|
||||
const predictions = await handPipeline.estimateHands(input, config);
|
||||
if (!predictions) return [];
|
||||
const hands: Array<Hand> = [];
|
||||
const hands: Array<HandResult> = [];
|
||||
for (let i = 0; i < predictions.length; i++) {
|
||||
const annotations = {};
|
||||
if (predictions[i].landmarks) {
|
||||
|
@ -72,8 +72,8 @@ export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
|
|||
box,
|
||||
boxRaw,
|
||||
keypoints,
|
||||
annotations: annotations as Hand['annotations'],
|
||||
landmarks: landmarks as Hand['landmarks'],
|
||||
annotations: annotations as HandResult['annotations'],
|
||||
landmarks: landmarks as HandResult['landmarks'],
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
|
@ -82,13 +82,13 @@ export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
|
|||
export async function load(config: Config): Promise<[GraphModel | null, GraphModel | null]> {
|
||||
if (!handDetectorModel || !handPoseModel) {
|
||||
[handDetectorModel, handPoseModel] = await Promise.all([
|
||||
config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) as unknown as GraphModel : null,
|
||||
config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) as unknown as GraphModel : null,
|
||||
config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector?.modelPath || ''), { fromTFHub: (config.hand.detector?.modelPath || '').includes('tfhub.dev') }) as unknown as GraphModel : null,
|
||||
config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton?.modelPath || ''), { fromTFHub: (config.hand.skeleton?.modelPath || '').includes('tfhub.dev') }) as unknown as GraphModel : null,
|
||||
]);
|
||||
if (config.hand.enabled) {
|
||||
if (!handDetectorModel || !handDetectorModel['modelUrl']) log('load model failed:', config.hand.detector.modelPath);
|
||||
if (!handDetectorModel || !handDetectorModel['modelUrl']) log('load model failed:', config.hand.detector?.modelPath || '');
|
||||
else if (config.debug) log('load model:', handDetectorModel['modelUrl']);
|
||||
if (!handPoseModel || !handPoseModel['modelUrl']) log('load model failed:', config.hand.skeleton.modelPath);
|
||||
if (!handPoseModel || !handPoseModel['modelUrl']) log('load model failed:', config.hand.skeleton?.modelPath || '');
|
||||
else if (config.debug) log('load model:', handPoseModel['modelUrl']);
|
||||
}
|
||||
} else {
|
||||
|
|
61
src/human.ts
61
src/human.ts
|
@ -4,7 +4,7 @@
|
|||
|
||||
import { log, now, mergeDeep } from './helpers';
|
||||
import { Config, defaults } from './config';
|
||||
import { Result, Face, Hand, Body, Item, Gesture } from './result';
|
||||
import { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult } from './result';
|
||||
import * as sysinfo from './sysinfo';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as backend from './tfjs/backend';
|
||||
|
@ -30,8 +30,9 @@ import * as app from '../package.json';
|
|||
import { Tensor, GraphModel } from './tfjs/types';
|
||||
|
||||
// export types
|
||||
export { Config } from './config';
|
||||
export type { Result, Face, Hand, Body, Item, Gesture, Person } from './result';
|
||||
export * from './config';
|
||||
export * from './result';
|
||||
|
||||
export type { DrawOptions } from './draw/draw';
|
||||
|
||||
/** Defines all possible input types for **Human** detection
|
||||
|
@ -101,16 +102,6 @@ export class Human {
|
|||
canvas: typeof draw.canvas,
|
||||
all: typeof draw.all,
|
||||
};
|
||||
/** Types used by Human */
|
||||
static Config: Config;
|
||||
static Result: Result;
|
||||
static Face: Face;
|
||||
static Hand: Hand;
|
||||
static Body: Body;
|
||||
static Item: Item;
|
||||
static Gesture: Gesture;
|
||||
static Person: Gesture
|
||||
static DrawOptions: draw.DrawOptions;
|
||||
/** @internal: Currently loaded models */
|
||||
models: {
|
||||
face: [unknown, GraphModel | null, GraphModel | null] | null,
|
||||
|
@ -529,10 +520,10 @@ export class Human {
|
|||
|
||||
// prepare where to store model results
|
||||
// keep them with weak typing as it can be promise or not
|
||||
let faceRes: Face[] | Promise<Face[]> | never[] = [];
|
||||
let bodyRes: Body[] | Promise<Body[]> | never[] = [];
|
||||
let handRes: Hand[] | Promise<Hand[]> | never[] = [];
|
||||
let objectRes: Item[] | Promise<Item[]> | never[] = [];
|
||||
let faceRes: FaceResult[] | Promise<FaceResult[]> | never[] = [];
|
||||
let bodyRes: BodyResult[] | Promise<BodyResult[]> | never[] = [];
|
||||
let handRes: HandResult[] | Promise<HandResult[]> | never[] = [];
|
||||
let objectRes: ObjectResult[] | Promise<ObjectResult[]> | never[] = [];
|
||||
|
||||
// run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
|
||||
if (this.config.async) {
|
||||
|
@ -549,18 +540,18 @@ export class Human {
|
|||
// run body: can be posenet, blazepose, efficientpose, movenet
|
||||
this.analyze('Start Body:');
|
||||
if (this.config.async) {
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.performance.body) delete this.performance.body;
|
||||
} else {
|
||||
this.state = 'run:body';
|
||||
timeStamp = now();
|
||||
if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.body = elapsedTime;
|
||||
}
|
||||
|
@ -583,14 +574,14 @@ export class Human {
|
|||
// run nanodet
|
||||
this.analyze('Start Object:');
|
||||
if (this.config.async) {
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.performance.object) delete this.performance.object;
|
||||
} else {
|
||||
this.state = 'run:object';
|
||||
timeStamp = now();
|
||||
if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : [];
|
||||
if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : [];
|
||||
else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : [];
|
||||
elapsedTime = Math.trunc(now() - timeStamp);
|
||||
if (elapsedTime > 0) this.performance.object = elapsedTime;
|
||||
}
|
||||
|
@ -600,7 +591,7 @@ export class Human {
|
|||
if (this.config.async) [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]);
|
||||
|
||||
// run gesture analysis last
|
||||
let gestureRes: Gesture[] = [];
|
||||
let gestureRes: GestureResult[] = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)];
|
||||
|
@ -612,15 +603,15 @@ export class Human {
|
|||
this.state = 'idle';
|
||||
const shape = this.process?.tensor?.shape || [];
|
||||
this.result = {
|
||||
face: faceRes as Face[],
|
||||
body: bodyRes as Body[],
|
||||
hand: handRes as Hand[],
|
||||
face: faceRes as FaceResult[],
|
||||
body: bodyRes as BodyResult[],
|
||||
hand: handRes as HandResult[],
|
||||
gesture: gestureRes,
|
||||
object: objectRes as Item[],
|
||||
object: objectRes as ObjectResult[],
|
||||
performance: this.performance,
|
||||
canvas: this.process.canvas,
|
||||
timestamp: Date.now(),
|
||||
get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); },
|
||||
get persons() { return persons.join(faceRes as FaceResult[], bodyRes as BodyResult[], handRes as HandResult[], gestureRes, shape); },
|
||||
};
|
||||
|
||||
// finally dispose input tensor
|
||||
|
|
|
@ -57,10 +57,10 @@ export function process(input: Input, config: Config): { tensor: Tensor | null,
|
|||
}
|
||||
|
||||
// create our canvas and resize it if needed
|
||||
if (config.filter.width > 0) targetWidth = config.filter.width;
|
||||
else if (config.filter.height > 0) targetWidth = originalWidth * (config.filter.height / originalHeight);
|
||||
if (config.filter.height > 0) targetHeight = config.filter.height;
|
||||
else if (config.filter.width > 0) targetHeight = originalHeight * (config.filter.width / originalWidth);
|
||||
if ((config.filter.width || 0) > 0) targetWidth = config.filter.width;
|
||||
else if ((config.filter.height || 0) > 0) targetWidth = originalWidth * ((config.filter.height || 0) / originalHeight);
|
||||
if ((config.filter.height || 0) > 0) targetHeight = config.filter.height;
|
||||
else if ((config.filter.width || 0) > 0) targetHeight = originalHeight * ((config.filter.width || 0) / originalWidth);
|
||||
if (!targetWidth || !targetHeight) throw new Error('Human: Input cannot determine dimension');
|
||||
if (!inCanvas || (inCanvas?.width !== targetWidth) || (inCanvas?.height !== targetHeight)) {
|
||||
inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
* Module that interpolates results for smoother animations
|
||||
*/
|
||||
|
||||
import type { Result, Face, Body, Hand, Item, Gesture, Person } from './result';
|
||||
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult } from './result';
|
||||
|
||||
const bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
|
||||
|
@ -26,7 +26,7 @@ export function calc(newResult: Result): Result {
|
|||
|
||||
// interpolate body results
|
||||
if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) {
|
||||
bufferedResult.body = JSON.parse(JSON.stringify(newResult.body as Body[])); // deep clone once
|
||||
bufferedResult.body = JSON.parse(JSON.stringify(newResult.body as BodyResult[])); // deep clone once
|
||||
} else {
|
||||
for (let i = 0; i < newResult.body.length; i++) {
|
||||
const box = newResult.body[i].box // update box
|
||||
|
@ -52,7 +52,7 @@ export function calc(newResult: Result): Result {
|
|||
|
||||
// interpolate hand results
|
||||
if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) {
|
||||
bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand as Hand[])); // deep clone once
|
||||
bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand as HandResult[])); // deep clone once
|
||||
} else {
|
||||
for (let i = 0; i < newResult.hand.length; i++) {
|
||||
const box = (newResult.hand[i].box// update box
|
||||
|
@ -69,13 +69,13 @@ export function calc(newResult: Result): Result {
|
|||
annotations[key] = newResult.hand[i].annotations[key]
|
||||
.map((val, j) => val.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor));
|
||||
}
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as Hand['annotations'] }; // shallow clone plus updated values
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as HandResult['annotations'] }; // shallow clone plus updated values
|
||||
}
|
||||
}
|
||||
|
||||
// interpolate face results
|
||||
if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) {
|
||||
bufferedResult.face = JSON.parse(JSON.stringify(newResult.face as Face[])); // deep clone once
|
||||
bufferedResult.face = JSON.parse(JSON.stringify(newResult.face as FaceResult[])); // deep clone once
|
||||
} else {
|
||||
for (let i = 0; i < newResult.face.length; i++) {
|
||||
const box = (newResult.face[i].box // update box
|
||||
|
@ -104,7 +104,7 @@ export function calc(newResult: Result): Result {
|
|||
|
||||
// interpolate object detection results
|
||||
if (!bufferedResult.object || (newResult.object.length !== bufferedResult.object.length)) {
|
||||
bufferedResult.object = JSON.parse(JSON.stringify(newResult.object as Item[])); // deep clone once
|
||||
bufferedResult.object = JSON.parse(JSON.stringify(newResult.object as ObjectResult[])); // deep clone once
|
||||
} else {
|
||||
for (let i = 0; i < newResult.object.length; i++) {
|
||||
const box = (newResult.object[i].box // update box
|
||||
|
@ -119,7 +119,7 @@ export function calc(newResult: Result): Result {
|
|||
if (newResult.persons) {
|
||||
const newPersons = newResult.persons; // trigger getter function
|
||||
if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) {
|
||||
bufferedResult.persons = JSON.parse(JSON.stringify(newPersons as Person[]));
|
||||
bufferedResult.persons = JSON.parse(JSON.stringify(newPersons as PersonResult[]));
|
||||
} else {
|
||||
for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow
|
||||
bufferedResult.persons[i].box = (newPersons[i].box
|
||||
|
@ -129,7 +129,7 @@ export function calc(newResult: Result): Result {
|
|||
}
|
||||
|
||||
// just copy latest gestures without interpolation
|
||||
if (newResult.gesture) bufferedResult.gesture = newResult.gesture as Gesture[];
|
||||
if (newResult.gesture) bufferedResult.gesture = newResult.gesture as GestureResult[];
|
||||
if (newResult.performance) bufferedResult.performance = newResult.performance;
|
||||
|
||||
return bufferedResult;
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { Body } from '../result';
|
||||
import { BodyResult } from '../result';
|
||||
import { GraphModel, Tensor } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
|
@ -23,7 +23,7 @@ const bodyParts = ['nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftSh
|
|||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)) as unknown as GraphModel;
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath || '')) as unknown as GraphModel;
|
||||
if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath);
|
||||
else if (config.debug) log('load model:', model['modelUrl']);
|
||||
} else if (config.debug) log('cached model:', model['modelUrl']);
|
||||
|
@ -114,8 +114,8 @@ async function parseMultiPose(res, config, image) {
|
|||
return persons;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config): Promise<Body[]> {
|
||||
if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
|
||||
if ((skipped < (config.body.skipFrames || 0)) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
skipped++;
|
||||
return [{ id: 0, score, box, boxRaw, keypoints }];
|
||||
}
|
||||
|
|
|
@ -5,17 +5,17 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import { Item } from '../result';
|
||||
import { ObjectResult } from '../result';
|
||||
import { GraphModel, Tensor } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
let model;
|
||||
let last: Item[] = [];
|
||||
let last: ObjectResult[] = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath));
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath || ''));
|
||||
const inputs = Object.values(model.modelSignature['inputs']);
|
||||
model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
|
||||
|
@ -27,7 +27,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
|
||||
async function process(res: Tensor, inputSize, outputShape, config: Config) {
|
||||
if (!res) return [];
|
||||
const results: Array<Item> = [];
|
||||
const results: Array<ObjectResult> = [];
|
||||
const detections = await res.array();
|
||||
const squeezeT = tf.squeeze(res);
|
||||
tf.dispose(res);
|
||||
|
@ -70,8 +70,8 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<Item[]> {
|
||||
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
|
||||
export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> {
|
||||
if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) {
|
||||
skipped++;
|
||||
return last;
|
||||
}
|
||||
|
|
|
@ -5,19 +5,19 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import { Item } from '../result';
|
||||
import { ObjectResult } from '../result';
|
||||
import { GraphModel, Tensor } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
let model;
|
||||
let last: Array<Item> = [];
|
||||
let last: Array<ObjectResult> = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
const scaleBox = 2.5; // increase box size
|
||||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath));
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath || ''));
|
||||
const inputs = Object.values(model.modelSignature['inputs']);
|
||||
model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null;
|
||||
if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`);
|
||||
|
@ -29,7 +29,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
|
||||
async function process(res, inputSize, outputShape, config) {
|
||||
let id = 0;
|
||||
let results: Array<Item> = [];
|
||||
let results: Array<ObjectResult> = [];
|
||||
for (const strideSize of [1, 2, 4]) { // try each stride size as it detects large/medium/small objects
|
||||
// find scores, boxes, classes
|
||||
tf.tidy(async () => { // wrap in tidy to automatically deallocate temp tensors
|
||||
|
@ -102,8 +102,8 @@ async function process(res, inputSize, outputShape, config) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config): Promise<Item[]> {
|
||||
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
|
||||
export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> {
|
||||
if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) {
|
||||
skipped++;
|
||||
return last;
|
||||
}
|
||||
|
|
|
@ -2,13 +2,13 @@
|
|||
* Module that analyzes existing results and recombines them into a unified person object
|
||||
*/
|
||||
|
||||
import { Face, Body, Hand, Gesture, Person } from './result';
|
||||
import { FaceResult, BodyResult, HandResult, GestureResult, PersonResult } from './result';
|
||||
|
||||
export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>, gestures: Array<Gesture>, shape: Array<number> | undefined): Array<Person> {
|
||||
export function join(faces: Array<FaceResult>, bodies: Array<BodyResult>, hands: Array<HandResult>, gestures: Array<GestureResult>, shape: Array<number> | undefined): Array<PersonResult> {
|
||||
let id = 0;
|
||||
const persons: Array<Person> = [];
|
||||
const persons: Array<PersonResult> = [];
|
||||
for (const face of faces) { // person is defined primarily by face and then we append other objects as found
|
||||
const person: Person = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] };
|
||||
const person: PersonResult = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] };
|
||||
for (const body of bodies) {
|
||||
if (face.box[0] > body.box[0] // x within body
|
||||
&& face.box[0] < body.box[0] + body.box[2]
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import * as kpt from './keypoints';
|
||||
import { Body } from '../result';
|
||||
import { BodyResult } from '../result';
|
||||
|
||||
export function eitherPointDoesntMeetConfidence(a: number, b: number, minConfidence: number) {
|
||||
return (a < minConfidence || b < minConfidence);
|
||||
|
@ -30,7 +30,7 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
|
|||
return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY];
|
||||
}
|
||||
|
||||
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<Body> {
|
||||
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<BodyResult> {
|
||||
const scaleY = height / inputResolutionHeight;
|
||||
const scaleX = width / inputResolutionWidth;
|
||||
const scalePose = (pose, i) => ({
|
||||
|
|
|
@ -32,7 +32,7 @@ import { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/ge
|
|||
* - gaze: gaze direction as object with values for bearing in radians and relative strength
|
||||
* - tensor: face tensor as Tensor object which contains detected face
|
||||
*/
|
||||
export interface Face {
|
||||
export interface FaceResult {
|
||||
id: number
|
||||
score: number,
|
||||
boxScore: number,
|
||||
|
@ -69,7 +69,7 @@ export interface Face {
|
|||
* - score: body part score value
|
||||
* - presence: body part presence value
|
||||
*/
|
||||
export interface Body {
|
||||
export interface BodyResult {
|
||||
id: number,
|
||||
score: number,
|
||||
box: [number, number, number, number],
|
||||
|
@ -94,7 +94,7 @@ export interface Body {
|
|||
* - annotations: annotated landmarks for each hand part with keypoints
|
||||
* - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
|
||||
*/
|
||||
export interface Hand {
|
||||
export interface HandResult {
|
||||
id: number,
|
||||
score: number,
|
||||
box: [number, number, number, number],
|
||||
|
@ -122,7 +122,7 @@ export interface Hand {
|
|||
* - center: optional center point as array of [x, y], normalized to image resolution
|
||||
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
|
||||
*/
|
||||
export interface Item {
|
||||
export interface ObjectResult {
|
||||
id: number,
|
||||
score: number,
|
||||
class: number,
|
||||
|
@ -139,7 +139,7 @@ export interface Item {
|
|||
* - part: part name and number where gesture was detected: face, iris, body, hand
|
||||
* - gesture: gesture detected
|
||||
*/
|
||||
export type Gesture =
|
||||
export type GestureResult =
|
||||
{ 'face': number, gesture: FaceGesture }
|
||||
| { 'iris': number, gesture: IrisGesture }
|
||||
| { 'body': number, gesture: BodyGesture }
|
||||
|
@ -157,12 +157,12 @@ export type Gesture =
|
|||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
*/
|
||||
export interface Person {
|
||||
export interface PersonResult {
|
||||
id: number,
|
||||
face: Face,
|
||||
body: Body | null,
|
||||
hands: { left: Hand | null, right: Hand | null },
|
||||
gestures: Array<Gesture>,
|
||||
face: FaceResult,
|
||||
body: BodyResult | null,
|
||||
hands: { left: HandResult | null, right: HandResult | null },
|
||||
gestures: Array<GestureResult>,
|
||||
box: [number, number, number, number],
|
||||
boxRaw?: [number, number, number, number],
|
||||
}
|
||||
|
@ -173,16 +173,16 @@ export interface Person {
|
|||
* Contains all possible detection results
|
||||
*/
|
||||
export interface Result {
|
||||
/** {@link Face}: detection & analysis results */
|
||||
face: Array<Face>,
|
||||
/** {@link Body}: detection & analysis results */
|
||||
body: Array<Body>,
|
||||
/** {@link Hand}: detection & analysis results */
|
||||
hand: Array<Hand>,
|
||||
/** {@link Gesture}: detection & analysis results */
|
||||
gesture: Array<Gesture>,
|
||||
/** {@link Object}: detection & analysis results */
|
||||
object: Array<Item>
|
||||
/** {@link FaceResult}: detection & analysis results */
|
||||
face: Array<FaceResult>,
|
||||
/** {@link BodyResult}: detection & analysis results */
|
||||
body: Array<BodyResult>,
|
||||
/** {@link HandResult}: detection & analysis results */
|
||||
hand: Array<HandResult>,
|
||||
/** {@link GestureResult}: detection & analysis results */
|
||||
gesture: Array<GestureResult>,
|
||||
/** {@link ItemResult}: detection & analysis results */
|
||||
object: Array<ObjectResult>
|
||||
/** global performance object with timing values for each operation */
|
||||
performance: Record<string, unknown>,
|
||||
/** optional processed canvas that can be used to draw input on screen */
|
||||
|
@ -190,5 +190,5 @@ export interface Result {
|
|||
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
|
||||
readonly timestamp: number,
|
||||
/** getter property that returns unified persons object */
|
||||
persons: Array<Person>,
|
||||
persons: Array<PersonResult>,
|
||||
}
|
||||
|
|
|
@ -15,7 +15,7 @@ let busy = false;
|
|||
|
||||
export async function load(config: Config): Promise<GraphModel> {
|
||||
if (!model) {
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.segmentation.modelPath)) as unknown as GraphModel;
|
||||
model = await tf.loadGraphModel(join(config.modelBasePath, config.segmentation.modelPath || '')) as unknown as GraphModel;
|
||||
if (!model || !model['modelUrl']) log('load model failed:', config.segmentation.modelPath);
|
||||
else if (config.debug) log('load model:', model['modelUrl']);
|
||||
} else if (config.debug) log('cached model:', model['modelUrl']);
|
||||
|
|
Loading…
Reference in New Issue