diff --git a/README.md b/README.md index 5806289d..2b5866d6 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) ap - [**Usage & Functions**](https://github.com/vladmandic/human/wiki/Usage) - [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration) - [**Output Details**](https://github.com/vladmandic/human/wiki/Outputs) +- [**Caching & Smoothing**](https://github.com/vladmandic/human/wiki/Caching) - [**Face Recognition & Face Description**](https://github.com/vladmandic/human/wiki/Embedding) - [**Gesture Recognition**](https://github.com/vladmandic/human/wiki/Gesture) - [**Common Issues**](https://github.com/vladmandic/human/wiki/Issues) diff --git a/TODO.md b/TODO.md index 9c405006..e16da45a 100644 --- a/TODO.md +++ b/TODO.md @@ -2,16 +2,14 @@ ## Work in Progress -- `skipTime`: testing, documentation -
### Exploring -- Optical Flow: -- TFLite Models: -- Histogram Equalization: Regular, Adaptive, Contrast Limited - Switch to custom `tfjs` for main `human` ESM bundle +- Optical Flow: +- Histogram Equalization: Regular, Adaptive, Contrast Limited +- TFLite Models: - Body segmentation: `robust-video-matting` #### WebGPU @@ -55,6 +53,7 @@ Object detection using CenterNet or NanoDet models is not working when using WAS ## Pending Release - Update to TFJS 3.10.0 +- Time based caching - Multiple bug fixes - Utility class `human.env` - Add `skipTime` in addition to `skipFrames` diff --git a/demo/index.js b/demo/index.js index e358eebe..1875f21f 100644 --- a/demo/index.js +++ b/demo/index.js @@ -1016,6 +1016,7 @@ async function main() { // create instance of human human = new Human(userConfig); + human.env.perfadd = true; log('human version:', human.version); // we've merged human defaults with user config and now lets store it back so it can be accessed by methods such as menu diff --git a/src/body/blazepose.ts b/src/body/blazepose.ts index 0eda79a8..3135e145 100644 --- a/src/body/blazepose.ts +++ b/src/body/blazepose.ts @@ -16,7 +16,7 @@ let skipped = Number.MAX_SAFE_INTEGER; let outputNodes: string[]; // different for lite/full/heavy let cache: BodyResult | null = null; let padding: [number, number][] = [[0, 0], [0, 0], [0, 0], [0, 0]]; -let last = 0; +let lastTime = 0; export async function loadDetect(config: Config): Promise { if (env.initial) models[0] = null; @@ -136,11 +136,13 @@ async function detectParts(input: Tensor, config: Config, outputSize: [number, n export async function predict(input: Tensor, config: Config): Promise { const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0]; - if ((skipped < (config.body.skipFrames || 0)) && ((config.body.skipTime || 0) <= (now() - last)) && config.skipFrame && cache !== null) { + const skipTime = (config.body.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.body.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && cache !== null) { skipped++; } else { cache = await detectParts(input, config, outputSize); - last = now(); + lastTime = now(); skipped = 0; } if (cache) return [cache]; diff --git a/src/body/efficientpose.ts b/src/body/efficientpose.ts index 980c0e13..3d88d9db 100644 --- a/src/body/efficientpose.ts +++ b/src/body/efficientpose.ts @@ -13,7 +13,7 @@ import type { Config } from '../config'; import { env } from '../util/env'; let model: GraphModel | null; -let last = 0; +let lastTime = 0; const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} }; // const keypoints: Array = []; @@ -50,7 +50,9 @@ function max2d(inputs, minScore) { } export async function predict(image: Tensor, config: Config): Promise { - if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0 && ((config.body.skipTime || 0) <= (now() - last))) { + const skipTime = (config.body.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.body.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && Object.keys(cache.keypoints).length > 0) { skipped++; return [cache]; } @@ -66,7 +68,7 @@ export async function predict(image: Tensor, config: Config): Promise { if (!model || !model?.inputs[0].shape) return []; // something is wrong with the model - if (!config.skipFrame) cache.boxes.length = 0; // allowed to use cache or not + if (!config.skipAllowed) cache.boxes.length = 0; // allowed to use cache or not skipped++; // increment skip frames - if (config.skipFrame && (skipped <= (config.body.skipFrames || 0) && ((config.body.skipTime || 0) <= (now() - cache.last)))) { + const skipTime = (config.body.skipTime || 0) > (now() - cache.last); + const skipFrame = skipped < (config.body.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame) { return cache.bodies; // return cached results without running anything } return new Promise(async (resolve) => { diff --git a/src/config.ts b/src/config.ts index e10a9dff..5f03b521 100644 --- a/src/config.ts +++ b/src/config.ts @@ -255,7 +255,7 @@ export interface Config { cacheSensitivity: number; /** Internal Variable */ - skipFrame: boolean; + skipAllowed: boolean; /** Run input through image filters before inference * - image filters run with near-zero latency as they are executed on the GPU @@ -302,10 +302,10 @@ const config: Config = { // warmup pre-initializes all models for faster inference but can take // significant time on startup // only used for `webgl` and `humangl` backends - cacheSensitivity: 0.75, // cache sensitivity + cacheSensitivity: 0.70, // cache sensitivity // values 0..1 where 0.01 means reset cache if input changed more than 1% // set to 0 to disable caching - skipFrame: false, // internal & dynamic + skipAllowed: false, // internal & dynamic filter: { // run input through image filters before inference // image filters run with near-zero latency as they are executed on the GPU enabled: true, // enable image pre-processing filters @@ -347,9 +347,9 @@ const config: Config = { // this parameter is not valid in nodejs maxDetected: 1, // maximum number of faces detected in the input // should be set to the minimum number for performance - skipFrames: 11, // how many max frames to go without re-running the face bounding box detector + skipFrames: 99, // how many max frames to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 2500, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero minConfidence: 0.2, // threshold for discarding a prediction iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed @@ -371,9 +371,9 @@ const config: Config = { emotion: { enabled: true, minConfidence: 0.1, // threshold for discarding a prediction - skipFrames: 12, // how max many frames to go without re-running the detector + skipFrames: 99, // how max many frames to go without re-running the detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 1500, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath }, @@ -383,18 +383,18 @@ const config: Config = { // recommended to enable detector.rotation and mesh.enabled modelPath: 'faceres.json', // face description model // can be either absolute path or relative to modelBasePath - skipFrames: 13, // how many max frames to go without re-running the detector + skipFrames: 99, // how many max frames to go without re-running the detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 3000, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero minConfidence: 0.1, // threshold for discarding a prediction }, antispoof: { enabled: false, - skipFrames: 14, // how max many frames to go without re-running the detector + skipFrames: 99, // how max many frames to go without re-running the detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 4000, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero modelPath: 'antispoof.json', // face description model // can be either absolute path or relative to modelBasePath @@ -415,7 +415,7 @@ const config: Config = { minConfidence: 0.3, // threshold for discarding a prediction skipFrames: 1, // how many max frames to go without re-running the detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 200, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero }, @@ -424,7 +424,7 @@ const config: Config = { rotation: true, // use best-guess rotated hand image or just box with rotation as-is // false means higher performance, but incorrect finger mapping if hand is inverted // only valid for `handdetect` variation - skipFrames: 2, // how many max frames to go without re-running the hand bounding box detector + skipFrames: 99, // how many max frames to go without re-running the hand bounding box detector // only used when cacheSensitivity is not zero skipTime: 2000, // how many ms to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero @@ -450,9 +450,9 @@ const config: Config = { minConfidence: 0.2, // threshold for discarding a prediction iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed maxDetected: 10, // maximum number of objects detected in the input - skipFrames: 15, // how many max frames to go without re-running the detector + skipFrames: 99, // how many max frames to go without re-running the detector // only used when cacheSensitivity is not zero - skipTime: 2000, // how many ms to go without re-running the face bounding box detector + skipTime: 1000, // how many ms to go without re-running object detector // only used when cacheSensitivity is not zero }, diff --git a/src/face/antispoof.ts b/src/face/antispoof.ts index 780fc240..d72fdff9 100644 --- a/src/face/antispoof.ts +++ b/src/face/antispoof.ts @@ -12,7 +12,7 @@ let model: GraphModel | null; const cached: Array = []; let skipped = Number.MAX_SAFE_INTEGER; let lastCount = 0; -let last = 0; +let lastTime = 0; export async function load(config: Config): Promise { if (env.initial) model = null; @@ -26,7 +26,9 @@ export async function load(config: Config): Promise { export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; - if ((skipped < (config.face.antispoof?.skipFrames || 0)) && ((config.face.antispoof?.skipTime || 0) <= (now() - last)) && config.skipFrame && (lastCount === count) && cached[idx]) { + const skipTime = (config.face.antispoof?.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.face.antispoof?.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && (lastCount === count) && cached[idx]) { skipped++; return cached[idx]; } @@ -37,7 +39,7 @@ export async function predict(image: Tensor, config: Config, idx, count) { const num = (await res.data())[0]; cached[idx] = Math.round(100 * num) / 100; lastCount = count; - last = now(); + lastTime = now(); tf.dispose([resize, res]); resolve(cached[idx]); }); diff --git a/src/face/face.ts b/src/face/face.ts index add1b255..d733f3bf 100644 --- a/src/face/face.ts +++ b/src/face/face.ts @@ -4,6 +4,7 @@ */ import { log, now } from '../util/util'; +import { env } from '../util/env'; import * as tf from '../../dist/tfjs.esm.js'; import * as facemesh from './facemesh'; import * as emotion from '../gear/emotion'; @@ -29,7 +30,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): timeStamp = now(); const faces = await facemesh.predict(input, parent.config); - parent.performance.face = Math.trunc(now() - timeStamp); + parent.performance.face = env.perfadd ? (parent.performance.face || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); if (!input.shape || input.shape.length !== 4) return []; if (!faces) return []; // for (const face of faces) { @@ -53,7 +54,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): parent.state = 'run:emotion'; timeStamp = now(); emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null; - parent.performance.emotion = Math.trunc(now() - timeStamp); + parent.performance.emotion = env.perfadd ? (parent.performance.emotion || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } parent.analyze('End Emotion:'); @@ -65,7 +66,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): parent.state = 'run:antispoof'; timeStamp = now(); antispoofRes = parent.config.face.antispoof.enabled ? await antispoof.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null; - parent.performance.antispoof = Math.trunc(now() - timeStamp); + parent.performance.antispoof = env.perfadd ? (parent.performance.antispoof || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } parent.analyze('End AntiSpoof:'); @@ -91,7 +92,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): parent.state = 'run:description'; timeStamp = now(); descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null; - parent.performance.embedding = Math.trunc(now() - timeStamp); + parent.performance.embedding = env.perfadd ? (parent.performance.embedding || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } parent.analyze('End Description:'); diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index 7e7efdee..cfa62cb2 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -28,7 +28,10 @@ let detectedFaces = 0; export async function predict(input: Tensor, config: Config): Promise { // reset cached boxes - if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)))) { + + const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.face.detector?.skipFrames || 0); + if (!config.skipAllowed || !skipTime || !skipFrame || detectedFaces === 0) { const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector lastTime = now(); boxCache = []; // empty cache diff --git a/src/face/faceres.ts b/src/face/faceres.ts index ef41cd75..21407dbf 100644 --- a/src/face/faceres.ts +++ b/src/face/faceres.ts @@ -91,7 +91,9 @@ export function enhance(input): Tensor { export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; - if ((skipped < (config.face.description?.skipFrames || 0)) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { + const skipFrame = skipped < (config.face.description?.skipFrames || 0); + const skipTime = (config.face.description?.skipTime || 0) > (now() - lastTime); + if (config.skipAllowed && skipFrame && skipTime && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { skipped++; return last[idx]; } diff --git a/src/gear/emotion.ts b/src/gear/emotion.ts index f5f191d9..cb575a87 100644 --- a/src/gear/emotion.ts +++ b/src/gear/emotion.ts @@ -33,7 +33,9 @@ export async function load(config: Config): Promise { export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; - if ((skipped < (config.face.emotion?.skipFrames || 0)) && ((config.face.emotion?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) { + const skipFrame = skipped < (config.face.emotion?.skipFrames || 0); + const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime); + if (config.skipAllowed && skipTime && skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) { skipped++; return last[idx]; } diff --git a/src/gear/ssrnet-age.ts b/src/gear/ssrnet-age.ts index 01467c6c..90e0dc56 100644 --- a/src/gear/ssrnet-age.ts +++ b/src/gear/ssrnet-age.ts @@ -33,7 +33,9 @@ export async function load(config: Config | any) { // eslint-disable-next-line @typescript-eslint/no-explicit-any export async function predict(image: Tensor, config: Config | any) { if (!model) return null; - if ((skipped < config.face.age.skipFrames) && ((config.face.age.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.age && (last.age > 0)) { + const skipTime = (config.face.age?.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.face.age?.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && last.age && (last.age > 0)) { skipped++; return last; } diff --git a/src/gear/ssrnet-gender.ts b/src/gear/ssrnet-gender.ts index 6957c835..2841d1e1 100644 --- a/src/gear/ssrnet-gender.ts +++ b/src/gear/ssrnet-gender.ts @@ -36,7 +36,9 @@ export async function load(config: Config | any) { // eslint-disable-next-line @typescript-eslint/no-explicit-any export async function predict(image: Tensor, config: Config | any) { if (!model) return null; - if ((skipped < config.face.gender.skipFrames) && ((config.face.gender.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.gender !== '') { + const skipTime = (config.face.gender?.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.face.gender?.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && last.gender !== '') { skipped++; return last; } diff --git a/src/hand/handposepipeline.ts b/src/hand/handposepipeline.ts index 7e1d548b..3a7137e1 100644 --- a/src/hand/handposepipeline.ts +++ b/src/hand/handposepipeline.ts @@ -30,7 +30,7 @@ export class HandPipeline { this.handPoseModel = handPoseModel; this.inputSize = this.handPoseModel && this.handPoseModel.inputs[0].shape ? this.handPoseModel.inputs[0].shape[2] : 0; this.storedBoxes = []; - this.skipped = 0; + this.skipped = Number.MAX_SAFE_INTEGER; this.detectedHands = 0; } @@ -88,15 +88,15 @@ export class HandPipeline { async estimateHands(image, config) { let useFreshBox = false; - // run new detector every skipFrames unless we only want box to start with + // run new detector every skipFrames let boxes; - - // console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector? - if ((this.skipped === 0) || ((this.skipped > config.hand.skipFrames) && ((config.hand.skipTime || 0) <= (now() - lastTime))) || !config.hand.landmarks || !config.skipFrame) { + const skipTime = (config.hand.skipTime || 0) > (now() - lastTime); + const skipFrame = this.skipped < (config.hand.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame) { boxes = await this.handDetector.estimateHandBounds(image, config); this.skipped = 0; } - if (config.skipFrame) this.skipped++; + if (config.skipAllowed) this.skipped++; // if detector result count doesn't match current working set, use it to reset current working set if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) { diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index 1802cffe..7c9c007d 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -28,7 +28,7 @@ const boxExpandFact = 1.6; const maxDetectorResolution = 512; const detectorExpandFact = 1.4; -let skipped = 0; +let skipped = Number.MAX_SAFE_INTEGER; let lastTime = 0; let outputSize: [number, number] = [0, 0]; @@ -183,23 +183,20 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config) } export async function predict(input: Tensor, config: Config): Promise { - /** handtrack caching - * 1. if skipFrame returned cached - * 2. if any cached results but although not sure if its enough we continute anyhow for 3x skipframes - * 3. if not skipframe or eventually rerun detector to generated new cached boxes and reset skipped - * 4. generate cached boxes based on detected keypoints - */ if (!models[0] || !models[1] || !models[0]?.inputs[0].shape || !models[1]?.inputs[0].shape) return []; // something is wrong with the model outputSize = [input.shape[2] || 0, input.shape[1] || 0]; - skipped++; // increment skip frames - if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0)) && ((config.hand.skipTime || 0) <= (now() - lastTime))) { + const skipTime = (config.hand.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.hand.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame) { return cache.hands; // return cached results without running anything } return new Promise(async (resolve) => { - if (config.skipFrame && cache.hands.length === config.hand.maxDetected) { // we have all detected hands + const skipTimeExtended = 3 * (config.hand.skipTime || 0) > (now() - lastTime); + const skipFrameExtended = skipped < 3 * (config.hand.skipFrames || 0); + if (config.skipAllowed && cache.hands.length === config.hand.maxDetected) { // we have all detected hands so we're definitely skipping cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config))); - } else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && ((config.hand.skipTime || 0) <= 3 * (now() - lastTime)) && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer + } else if (config.skipAllowed && skipTimeExtended && skipFrameExtended && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config))); } else { // finally rerun detector cache.boxes = await detectHands(input, config); diff --git a/src/human.ts b/src/human.ts index d281aabe..736792f2 100644 --- a/src/human.ts +++ b/src/human.ts @@ -326,7 +326,7 @@ export class Human { const count = Object.values(this.models).filter((model) => model).length; if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config; - if (env.initial) { // print version info on first run and check for correct backend setup + if (this.env.initial) { // print version info on first run and check for correct backend setup if (this.config.debug) log(`version: ${this.version}`); if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`); if (!await backend.check(this)) log('error: backend check failed'); @@ -338,8 +338,8 @@ export class Human { } await models.load(this); // actually loads models - if (env.initial && this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors'); // print memory stats on first run - env.initial = false; + if (this.env.initial && this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors'); // print memory stats on first run + this.env.initial = false; const loaded = Object.values(this.models).filter((model) => model).length; if (loaded !== count) { // number of loaded models changed @@ -348,7 +348,7 @@ export class Human { } const current = Math.trunc(now() - timeStamp); - if (current > (this.performance.load as number || 0)) this.performance.load = current; + if (current > (this.performance.load as number || 0)) this.performance.load = this.env.perfadd ? (this.performance.load || 0) + current : current; } // emit event @@ -393,7 +393,6 @@ export class Human { return new Promise(async (resolve) => { this.state = 'config'; let timeStamp; - let elapsedTime; // update configuration this.config = mergeDeep(this.config, userConfig) as Config; @@ -418,7 +417,7 @@ export class Human { this.state = 'image'; const img = image.process(input, this.config) as { canvas: HTMLCanvasElement | OffscreenCanvas, tensor: Tensor }; this.process = img; - this.performance.image = Math.trunc(now() - timeStamp); + this.performance.image = this.env.perfadd ? (this.performance.image || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); this.analyze('Get Image:'); if (!img.tensor) { @@ -429,12 +428,12 @@ export class Human { this.emit('image'); timeStamp = now(); - this.config.skipFrame = await image.skip(this.config, img.tensor); + this.config.skipAllowed = await image.skip(this.config, img.tensor); if (!this.performance.frames) this.performance.frames = 0; if (!this.performance.cached) this.performance.cached = 0; (this.performance.frames as number)++; - if (this.config.skipFrame) this.performance.cached++; - this.performance.changed = Math.trunc(now() - timeStamp); + if (this.config.skipAllowed) this.performance.cached++; + this.performance.changed = this.env.perfadd ? (this.performance.changed || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); this.analyze('Check Changed:'); // prepare where to store model results @@ -452,8 +451,7 @@ export class Human { } else { timeStamp = now(); faceRes = this.config.face.enabled ? await face.detectFace(this, img.tensor) : []; - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.face = elapsedTime; + this.performance.face = this.env.perfadd ? (this.performance.face || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } if (this.config.async && (this.config.body.maxDetected === -1 || this.config.hand.maxDetected === -1)) faceRes = await faceRes; // need face result for auto-detect number of hands or bodies @@ -474,8 +472,7 @@ export class Human { else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(img.tensor, bodyConfig) : []; else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(img.tensor, bodyConfig) : []; else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(img.tensor, bodyConfig) : []; - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.body = elapsedTime; + this.performance.body = this.env.perfadd ? (this.performance.body || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } this.analyze('End Body:'); @@ -491,8 +488,7 @@ export class Human { timeStamp = now(); if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? await handpose.predict(img.tensor, handConfig) : []; else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? await handtrack.predict(img.tensor, handConfig) : []; - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.hand = elapsedTime; + this.performance.hand = this.env.perfadd ? (this.performance.hand || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } this.analyze('End Hand:'); @@ -507,8 +503,7 @@ export class Human { timeStamp = now(); if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(img.tensor, this.config) : []; else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(img.tensor, this.config) : []; - elapsedTime = Math.trunc(now() - timeStamp); - if (elapsedTime > 0) this.performance.object = elapsedTime; + this.performance.object = this.env.perfadd ? (this.performance.object || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); } this.analyze('End Object:'); @@ -522,7 +517,7 @@ export class Human { if (this.config.gesture.enabled) { timeStamp = now(); gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)]; - if (!this.config.async) this.performance.gesture = Math.trunc(now() - timeStamp); + if (!this.config.async) this.performance.gesture = this.env.perfadd ? (this.performance.gesture || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp); else if (this.performance.gesture) delete this.performance.gesture; } diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 39338df7..600de2b8 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -79,7 +79,9 @@ async function process(res: Tensor | null, outputShape, config: Config) { } export async function predict(input: Tensor, config: Config): Promise { - if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) { + const skipTime = (config.object.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.object.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && (last.length > 0)) { skipped++; return last; } diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 9f63c25a..67a1bb15 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -107,7 +107,9 @@ async function process(res, inputSize, outputShape, config) { } export async function predict(image: Tensor, config: Config): Promise { - if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) { + const skipTime = (config.object.skipTime || 0) > (now() - lastTime); + const skipFrame = skipped < (config.object.skipFrames || 0); + if (config.skipAllowed && skipTime && skipFrame && (last.length > 0)) { skipped++; return last; } diff --git a/src/result.ts b/src/result.ts index a90d96cd..26579a4b 100644 --- a/src/result.ts +++ b/src/result.ts @@ -194,7 +194,7 @@ export interface Result { /** {@link ObjectResult}: detection & analysis results */ object: Array /** global performance object with timing values for each operation */ - performance: Record, + performance: Record, /** optional processed canvas that can be used to draw input on screen */ canvas?: OffscreenCanvas | HTMLCanvasElement | null | undefined, /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ diff --git a/src/util/env.ts b/src/util/env.ts index 267eea30..474a7473 100644 --- a/src/util/env.ts +++ b/src/util/env.ts @@ -25,6 +25,8 @@ export class Env { }; /** Is offscreenCanvas supported? */ offscreen: undefined | boolean; + /** Are performance counter instant values or additive */ + perfadd: boolean = false; /** WASM detected capabilities */ wasm: { supported: undefined | boolean, diff --git a/wiki b/wiki index d6c2c8c4..97e86c65 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d6c2c8c474f1d55f36e4ab4ffc9a1852f2f2b4fb +Subproject commit 97e86c65f64df007c25250bcb513d48e5c602242