time based caching

README.md

@@ -60,6 +60,7 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) ap
 - [**Usage & Functions**](https://github.com/vladmandic/human/wiki/Usage)
 - [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration)
 - [**Output Details**](https://github.com/vladmandic/human/wiki/Outputs)
+- [**Caching & Smoothing**](https://github.com/vladmandic/human/wiki/Caching)
 - [**Face Recognition & Face Description**](https://github.com/vladmandic/human/wiki/Embedding)
 - [**Gesture Recognition**](https://github.com/vladmandic/human/wiki/Gesture)
 - [**Common Issues**](https://github.com/vladmandic/human/wiki/Issues)

TODO.md

@@ -2,16 +2,14 @@
 
 ## Work in Progress
 
-- `skipTime`: testing, documentation
-
 <br>
 
 ### Exploring
 
-- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
-- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
-- Histogram Equalization: Regular, Adaptive, Contrast Limited
 - Switch to custom `tfjs` for main `human` ESM bundle
+- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
+- Histogram Equalization: Regular, Adaptive, Contrast Limited
+- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
 - Body segmentation: `robust-video-matting`
 
 #### WebGPU
@@ -55,6 +53,7 @@ Object detection using CenterNet or NanoDet models is not working when using WAS
 ## Pending Release
 
 - Update to TFJS 3.10.0
+- Time based caching
 - Multiple bug fixes
 - Utility class `human.env`
 - Add `skipTime` in addition to `skipFrames`

demo/index.js

@@ -1016,6 +1016,7 @@ async function main() {
 
   // create instance of human
   human = new Human(userConfig);
+  human.env.perfadd = true;
 
   log('human version:', human.version);
   // we've merged human defaults with user config and now lets store it back so it can be accessed by methods such as menu

src/body/blazepose.ts

@@ -16,7 +16,7 @@ let skipped = Number.MAX_SAFE_INTEGER;
 let outputNodes: string[]; // different for lite/full/heavy
 let cache: BodyResult | null = null;
 let padding: [number, number][] = [[0, 0], [0, 0], [0, 0], [0, 0]];
-let last = 0;
+let lastTime = 0;
 
 export async function loadDetect(config: Config): Promise<GraphModel> {
   if (env.initial) models[0] = null;
@@ -136,11 +136,13 @@ async function detectParts(input: Tensor, config: Config, outputSize: [number, n
 
 export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
   const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0];
-  if ((skipped < (config.body.skipFrames || 0)) && ((config.body.skipTime || 0) <= (now() - last)) && config.skipFrame && cache !== null) {
+  const skipTime = (config.body.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.body.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && cache !== null) {
     skipped++;
   } else {
     cache = await detectParts(input, config, outputSize);
-    last = now();
+    lastTime = now();
     skipped = 0;
   }
   if (cache) return [cache];

src/body/efficientpose.ts

@@ -13,7 +13,7 @@ import type { Config } from '../config';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
-let last = 0;
+let lastTime = 0;
 const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} };
 
 // const keypoints: Array<BodyKeypoint> = [];
@@ -50,7 +50,9 @@ function max2d(inputs, minScore) {
 }
 
 export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
-  if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0 && ((config.body.skipTime || 0) <= (now() - last))) {
+  const skipTime = (config.body.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.body.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && Object.keys(cache.keypoints).length > 0) {
     skipped++;
     return [cache];
   }
@@ -66,7 +68,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
 
     let resT;
     if (config.body.enabled) resT = await model?.predict(tensor);
-    last = now();
+    lastTime = now();
     tf.dispose(tensor);
 
     if (resT) {

src/body/movenet.ts

@@ -132,9 +132,11 @@ async function parseMultiPose(res, config, image, inputBox) {
 
 export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
   if (!model || !model?.inputs[0].shape) return []; // something is wrong with the model
-  if (!config.skipFrame) cache.boxes.length = 0; // allowed to use cache or not
+  if (!config.skipAllowed) cache.boxes.length = 0; // allowed to use cache or not
   skipped++; // increment skip frames
-  if (config.skipFrame && (skipped <= (config.body.skipFrames || 0) && ((config.body.skipTime || 0) <= (now() - cache.last)))) {
+  const skipTime = (config.body.skipTime || 0) > (now() - cache.last);
+  const skipFrame = skipped < (config.body.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame) {
     return cache.bodies; // return cached results without running anything
   }
   return new Promise(async (resolve) => {

src/config.ts

@@ -255,7 +255,7 @@ export interface Config {
   cacheSensitivity: number;
 
   /** Internal Variable */
-  skipFrame: boolean;
+  skipAllowed: boolean;
 
   /** Run input through image filters before inference
    * - image filters run with near-zero latency as they are executed on the GPU
@@ -302,10 +302,10 @@ const config: Config = {
                              // warmup pre-initializes all models for faster inference but can take
                              // significant time on startup
                              // only used for `webgl` and `humangl` backends
-  cacheSensitivity: 0.75,    // cache sensitivity
+  cacheSensitivity: 0.70,    // cache sensitivity
                              // values 0..1 where 0.01 means reset cache if input changed more than 1%
                              // set to 0 to disable caching
-  skipFrame: false,          // internal & dynamic
+  skipAllowed: false,        // internal & dynamic
   filter: {                  // run input through image filters before inference
                              // image filters run with near-zero latency as they are executed on the GPU
     enabled: true,           // enable image pre-processing filters
@@ -347,9 +347,9 @@ const config: Config = {
                              // this parameter is not valid in nodejs
       maxDetected: 1,        // maximum number of faces detected in the input
                              // should be set to the minimum number for performance
-      skipFrames: 11,        // how many max frames to go without re-running the face bounding box detector
+      skipFrames: 99,        // how many max frames to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
-      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+      skipTime: 2500,        // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
       minConfidence: 0.2,    // threshold for discarding a prediction
       iouThreshold: 0.1,     // ammount of overlap between two detected objects before one object is removed
@@ -371,9 +371,9 @@ const config: Config = {
     emotion: {
       enabled: true,
       minConfidence: 0.1,    // threshold for discarding a prediction
-      skipFrames: 12,        // how max many frames to go without re-running the detector
+      skipFrames: 99,        // how max many frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
-      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+      skipTime: 1500,        // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
       modelPath: 'emotion.json',  // face emotion model, can be absolute path or relative to modelBasePath
     },
@@ -383,18 +383,18 @@ const config: Config = {
                              // recommended to enable detector.rotation and mesh.enabled
       modelPath: 'faceres.json',  // face description model
                              // can be either absolute path or relative to modelBasePath
-      skipFrames: 13,        // how many max frames to go without re-running the detector
+      skipFrames: 99,        // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
-      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+      skipTime: 3000,        // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
       minConfidence: 0.1,    // threshold for discarding a prediction
     },
 
     antispoof: {
       enabled: false,
-      skipFrames: 14,        // how max many frames to go without re-running the detector
+      skipFrames: 99,        // how max many frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
-      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+      skipTime: 4000,        // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
       modelPath: 'antispoof.json',  // face description model
                              // can be either absolute path or relative to modelBasePath
@@ -415,7 +415,7 @@ const config: Config = {
     minConfidence: 0.3,      // threshold for discarding a prediction
     skipFrames: 1,           // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
-    skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
+    skipTime: 200,           // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
 },
 
@@ -424,7 +424,7 @@ const config: Config = {
     rotation: true,          // use best-guess rotated hand image or just box with rotation as-is
                              // false means higher performance, but incorrect finger mapping if hand is inverted
                              // only valid for `handdetect` variation
-    skipFrames: 2,           // how many max frames to go without re-running the hand bounding box detector
+    skipFrames: 99,          // how many max frames to go without re-running the hand bounding box detector
                              // only used when cacheSensitivity is not zero
     skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
@@ -450,9 +450,9 @@ const config: Config = {
     minConfidence: 0.2,      // threshold for discarding a prediction
     iouThreshold: 0.4,       // ammount of overlap between two detected objects before one object is removed
     maxDetected: 10,         // maximum number of objects detected in the input
-    skipFrames: 15,          // how many max frames to go without re-running the detector
+    skipFrames: 99,          // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
-    skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
+    skipTime: 1000,          // how many ms to go without re-running object detector
                              // only used when cacheSensitivity is not zero
   },
 

src/face/antispoof.ts

@@ -12,7 +12,7 @@ let model: GraphModel | null;
 const cached: Array<number> = [];
 let skipped = Number.MAX_SAFE_INTEGER;
 let lastCount = 0;
-let last = 0;
+let lastTime = 0;
 
 export async function load(config: Config): Promise<GraphModel> {
   if (env.initial) model = null;
@@ -26,7 +26,9 @@ export async function load(config: Config): Promise<GraphModel> {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.antispoof?.skipFrames || 0)) && ((config.face.antispoof?.skipTime || 0) <= (now() - last)) && config.skipFrame && (lastCount === count) && cached[idx]) {
+  const skipTime = (config.face.antispoof?.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.face.antispoof?.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && (lastCount === count) && cached[idx]) {
     skipped++;
     return cached[idx];
   }
@@ -37,7 +39,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
     const num = (await res.data())[0];
     cached[idx] = Math.round(100 * num) / 100;
     lastCount = count;
-    last = now();
+    lastTime = now();
     tf.dispose([resize, res]);
     resolve(cached[idx]);
   });

src/face/face.ts

@@ -4,6 +4,7 @@
  */
 
 import { log, now } from '../util/util';
+import { env } from '../util/env';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as facemesh from './facemesh';
 import * as emotion from '../gear/emotion';
@@ -29,7 +30,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
   timeStamp = now();
 
   const faces = await facemesh.predict(input, parent.config);
-  parent.performance.face = Math.trunc(now() - timeStamp);
+  parent.performance.face = env.perfadd ? (parent.performance.face || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
   if (!input.shape || input.shape.length !== 4) return [];
   if (!faces) return [];
   // for (const face of faces) {
@@ -53,7 +54,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
       parent.state = 'run:emotion';
       timeStamp = now();
       emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null;
-      parent.performance.emotion = Math.trunc(now() - timeStamp);
+      parent.performance.emotion = env.perfadd ? (parent.performance.emotion || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
     }
     parent.analyze('End Emotion:');
 
@@ -65,7 +66,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
       parent.state = 'run:antispoof';
       timeStamp = now();
       antispoofRes = parent.config.face.antispoof.enabled ? await antispoof.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null;
-      parent.performance.antispoof = Math.trunc(now() - timeStamp);
+      parent.performance.antispoof = env.perfadd ? (parent.performance.antispoof || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
     }
     parent.analyze('End AntiSpoof:');
 
@@ -91,7 +92,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
       parent.state = 'run:description';
       timeStamp = now();
       descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : null;
-      parent.performance.embedding = Math.trunc(now() - timeStamp);
+      parent.performance.embedding = env.perfadd ? (parent.performance.embedding || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
     }
     parent.analyze('End Description:');
 

src/face/facemesh.ts

@@ -28,7 +28,10 @@ let detectedFaces = 0;
 
 export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
   // reset cached boxes
-  if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)))) {
+
+  const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.face.detector?.skipFrames || 0);
+  if (!config.skipAllowed || !skipTime || !skipFrame || detectedFaces === 0) {
     const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
     lastTime = now();
     boxCache = []; // empty cache

src/face/faceres.ts

@@ -91,7 +91,9 @@ export function enhance(input): Tensor {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.description?.skipFrames || 0)) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
+  const skipFrame = skipped < (config.face.description?.skipFrames || 0);
+  const skipTime = (config.face.description?.skipTime || 0) > (now() - lastTime);
+  if (config.skipAllowed && skipFrame && skipTime && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
     skipped++;
     return last[idx];
   }

src/gear/emotion.ts

@@ -33,7 +33,9 @@ export async function load(config: Config): Promise<GraphModel> {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.emotion?.skipFrames || 0)) && ((config.face.emotion?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
+  const skipFrame = skipped < (config.face.emotion?.skipFrames || 0);
+  const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime);
+  if (config.skipAllowed && skipTime && skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
     skipped++;
     return last[idx];
   }

src/gear/ssrnet-age.ts

@@ -33,7 +33,9 @@ export async function load(config: Config | any) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function predict(image: Tensor, config: Config | any) {
   if (!model) return null;
-  if ((skipped < config.face.age.skipFrames) && ((config.face.age.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.age && (last.age > 0)) {
+  const skipTime = (config.face.age?.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.face.age?.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && last.age && (last.age > 0)) {
     skipped++;
     return last;
   }

src/gear/ssrnet-gender.ts

@@ -36,7 +36,9 @@ export async function load(config: Config | any) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function predict(image: Tensor, config: Config | any) {
   if (!model) return null;
-  if ((skipped < config.face.gender.skipFrames) && ((config.face.gender.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.gender !== '') {
+  const skipTime = (config.face.gender?.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.face.gender?.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && last.gender !== '') {
     skipped++;
     return last;
   }

src/hand/handposepipeline.ts

@@ -30,7 +30,7 @@ export class HandPipeline {
     this.handPoseModel = handPoseModel;
     this.inputSize = this.handPoseModel && this.handPoseModel.inputs[0].shape ? this.handPoseModel.inputs[0].shape[2] : 0;
     this.storedBoxes = [];
-    this.skipped = 0;
+    this.skipped = Number.MAX_SAFE_INTEGER;
     this.detectedHands = 0;
   }
 
@@ -88,15 +88,15 @@ export class HandPipeline {
   async estimateHands(image, config) {
     let useFreshBox = false;
 
-    // run new detector every skipFrames unless we only want box to start with
+    // run new detector every skipFrames
     let boxes;
-
+    const skipTime = (config.hand.skipTime || 0) > (now() - lastTime);
-    // console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
+    const skipFrame = this.skipped < (config.hand.skipFrames || 0);
-    if ((this.skipped === 0) || ((this.skipped > config.hand.skipFrames) && ((config.hand.skipTime || 0) <= (now() - lastTime))) || !config.hand.landmarks || !config.skipFrame) {
+    if (config.skipAllowed && skipTime && skipFrame) {
       boxes = await this.handDetector.estimateHandBounds(image, config);
       this.skipped = 0;
     }
-    if (config.skipFrame) this.skipped++;
+    if (config.skipAllowed) this.skipped++;
 
     // if detector result count doesn't match current working set, use it to reset current working set
     if (boxes && (boxes.length > 0) && ((boxes.length !== this.detectedHands) && (this.detectedHands !== config.hand.maxDetected) || !config.hand.landmarks)) {

src/hand/handtrack.ts

@@ -28,7 +28,7 @@ const boxExpandFact = 1.6;
 const maxDetectorResolution = 512;
 const detectorExpandFact = 1.4;
 
-let skipped = 0;
+let skipped = Number.MAX_SAFE_INTEGER;
 let lastTime = 0;
 let outputSize: [number, number] = [0, 0];
 
@@ -183,23 +183,20 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
 }
 
 export async function predict(input: Tensor, config: Config): Promise<HandResult[]> {
-  /** handtrack caching
-   * 1. if skipFrame returned cached
-   * 2. if any cached results but although not sure if its enough we continute anyhow for 3x skipframes
-   * 3. if not skipframe or eventually rerun detector to generated new cached boxes and reset skipped
-   * 4. generate cached boxes based on detected keypoints
-   */
   if (!models[0] || !models[1] || !models[0]?.inputs[0].shape || !models[1]?.inputs[0].shape) return []; // something is wrong with the model
   outputSize = [input.shape[2] || 0, input.shape[1] || 0];
-
   skipped++; // increment skip frames
-  if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0)) && ((config.hand.skipTime || 0) <= (now() - lastTime))) {
+  const skipTime = (config.hand.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.hand.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame) {
     return cache.hands; // return cached results without running anything
   }
   return new Promise(async (resolve) => {
-    if (config.skipFrame && cache.hands.length === config.hand.maxDetected) { // we have all detected hands
+    const skipTimeExtended = 3 * (config.hand.skipTime || 0) > (now() - lastTime);
+    const skipFrameExtended = skipped < 3 * (config.hand.skipFrames || 0);
+    if (config.skipAllowed && cache.hands.length === config.hand.maxDetected) { // we have all detected hands so we're definitely skipping
       cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
-    } else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && ((config.hand.skipTime || 0) <= 3 * (now() - lastTime)) && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer
+    } else if (config.skipAllowed && skipTimeExtended && skipFrameExtended && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer
       cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
     } else { // finally rerun detector
       cache.boxes = await detectHands(input, config);

src/human.ts

@@ -326,7 +326,7 @@ export class Human {
     const count = Object.values(this.models).filter((model) => model).length;
     if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config;
 
-    if (env.initial) { // print version info on first run and check for correct backend setup
+    if (this.env.initial) { // print version info on first run and check for correct backend setup
       if (this.config.debug) log(`version: ${this.version}`);
       if (this.config.debug) log(`tfjs version: ${this.tf.version_core}`);
       if (!await backend.check(this)) log('error: backend check failed');
@@ -338,8 +338,8 @@ export class Human {
     }
 
     await models.load(this); // actually loads models
-    if (env.initial && this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors'); // print memory stats on first run
+    if (this.env.initial && this.config.debug) log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors'); // print memory stats on first run
-    env.initial = false;
+    this.env.initial = false;
 
     const loaded = Object.values(this.models).filter((model) => model).length;
     if (loaded !== count) { // number of loaded models changed
@@ -348,7 +348,7 @@ export class Human {
     }
 
     const current = Math.trunc(now() - timeStamp);
-    if (current > (this.performance.load as number || 0)) this.performance.load = current;
+    if (current > (this.performance.load as number || 0)) this.performance.load = this.env.perfadd ? (this.performance.load || 0) + current : current;
   }
 
   // emit event
@@ -393,7 +393,6 @@ export class Human {
     return new Promise(async (resolve) => {
       this.state = 'config';
       let timeStamp;
-      let elapsedTime;
 
       // update configuration
       this.config = mergeDeep(this.config, userConfig) as Config;
@@ -418,7 +417,7 @@ export class Human {
       this.state = 'image';
       const img = image.process(input, this.config) as { canvas: HTMLCanvasElement | OffscreenCanvas, tensor: Tensor };
       this.process = img;
-      this.performance.image = Math.trunc(now() - timeStamp);
+      this.performance.image = this.env.perfadd ? (this.performance.image || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
       this.analyze('Get Image:');
 
       if (!img.tensor) {
@@ -429,12 +428,12 @@ export class Human {
       this.emit('image');
 
       timeStamp = now();
-      this.config.skipFrame = await image.skip(this.config, img.tensor);
+      this.config.skipAllowed = await image.skip(this.config, img.tensor);
       if (!this.performance.frames) this.performance.frames = 0;
       if (!this.performance.cached) this.performance.cached = 0;
       (this.performance.frames as number)++;
-      if (this.config.skipFrame) this.performance.cached++;
+      if (this.config.skipAllowed) this.performance.cached++;
-      this.performance.changed = Math.trunc(now() - timeStamp);
+      this.performance.changed = this.env.perfadd ? (this.performance.changed || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
       this.analyze('Check Changed:');
 
       // prepare where to store model results
@@ -452,8 +451,7 @@ export class Human {
       } else {
         timeStamp = now();
         faceRes = this.config.face.enabled ? await face.detectFace(this, img.tensor) : [];
-        elapsedTime = Math.trunc(now() - timeStamp);
+        this.performance.face = this.env.perfadd ? (this.performance.face || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
-        if (elapsedTime > 0) this.performance.face = elapsedTime;
       }
 
       if (this.config.async && (this.config.body.maxDetected === -1 || this.config.hand.maxDetected === -1)) faceRes = await faceRes; // need face result for auto-detect number of hands or bodies
@@ -474,8 +472,7 @@ export class Human {
         else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(img.tensor, bodyConfig) : [];
         else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(img.tensor, bodyConfig) : [];
         else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(img.tensor, bodyConfig) : [];
-        elapsedTime = Math.trunc(now() - timeStamp);
+        this.performance.body = this.env.perfadd ? (this.performance.body || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
-        if (elapsedTime > 0) this.performance.body = elapsedTime;
       }
       this.analyze('End Body:');
 
@@ -491,8 +488,7 @@ export class Human {
         timeStamp = now();
         if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? await handpose.predict(img.tensor, handConfig) : [];
         else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? await handtrack.predict(img.tensor, handConfig) : [];
-        elapsedTime = Math.trunc(now() - timeStamp);
+        this.performance.hand = this.env.perfadd ? (this.performance.hand || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
-        if (elapsedTime > 0) this.performance.hand = elapsedTime;
       }
       this.analyze('End Hand:');
 
@@ -507,8 +503,7 @@ export class Human {
         timeStamp = now();
         if (this.config.object.modelPath?.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(img.tensor, this.config) : [];
         else if (this.config.object.modelPath?.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(img.tensor, this.config) : [];
-        elapsedTime = Math.trunc(now() - timeStamp);
+        this.performance.object = this.env.perfadd ? (this.performance.object || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
-        if (elapsedTime > 0) this.performance.object = elapsedTime;
       }
       this.analyze('End Object:');
 
@@ -522,7 +517,7 @@ export class Human {
       if (this.config.gesture.enabled) {
         timeStamp = now();
         gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)];
-        if (!this.config.async) this.performance.gesture = Math.trunc(now() - timeStamp);
+        if (!this.config.async) this.performance.gesture = this.env.perfadd ? (this.performance.gesture || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
         else if (this.performance.gesture) delete this.performance.gesture;
       }
 

src/object/centernet.ts

@@ -79,7 +79,9 @@ async function process(res: Tensor | null, outputShape, config: Config) {
 }
 
 export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> {
-  if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
+  const skipTime = (config.object.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.object.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && (last.length > 0)) {
     skipped++;
     return last;
   }

src/object/nanodet.ts

@@ -107,7 +107,9 @@ async function process(res, inputSize, outputShape, config) {
 }
 
 export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> {
-  if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
+  const skipTime = (config.object.skipTime || 0) > (now() - lastTime);
+  const skipFrame = skipped < (config.object.skipFrames || 0);
+  if (config.skipAllowed && skipTime && skipFrame && (last.length > 0)) {
     skipped++;
     return last;
   }

src/result.ts

@@ -194,7 +194,7 @@ export interface Result {
   /** {@link ObjectResult}: detection & analysis results */
   object: Array<ObjectResult>
   /** global performance object with timing values for each operation */
-  performance: Record<string, unknown>,
+  performance: Record<string, number>,
   /** optional processed canvas that can be used to draw input on screen */
   canvas?: OffscreenCanvas | HTMLCanvasElement | null | undefined,
   /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */

src/util/env.ts

@@ -25,6 +25,8 @@ export class Env {
   };
   /** Is offscreenCanvas supported? */
   offscreen: undefined | boolean;
+  /** Are performance counter instant values or additive */
+  perfadd: boolean = false;
   /** WASM detected capabilities */
   wasm: {
     supported: undefined | boolean,

wiki

@@ -1 +1 @@
-Subproject commit d6c2c8c474f1d55f36e4ab4ffc9a1852f2f2b4fb
+Subproject commit 97e86c65f64df007c25250bcb513d48e5c602242