initial work on skipTime

CHANGELOG.md

@@ -11,6 +11,9 @@
   
 ### **HEAD -> main** 2021/10/22 mandic00@live.com
 
+
+### **origin/main** 2021/10/22 mandic00@live.com
+
 - add optional autodetected custom wasm path
 
 ### **2.3.6** 2021/10/21 mandic00@live.com

build.json

@@ -137,6 +137,7 @@
         "format": "esm",
         "input": "tfjs/tf-browser.ts",
         "output": "dist/tfjs.esm.js",
+        "minify": true,
         "sourcemap": true,
         "external": ["fs", "os", "buffer", "util"]
       },

src/body/blazepose.ts

@@ -3,7 +3,7 @@
  */
 
 import * as tf from '../../dist/tfjs.esm.js';
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
@@ -16,6 +16,7 @@ let skipped = Number.MAX_SAFE_INTEGER;
 let outputNodes: string[]; // different for lite/full/heavy
 let cache: BodyResult | null = null;
 let padding: [number, number][] = [[0, 0], [0, 0], [0, 0], [0, 0]];
+let last = 0;
 
 export async function loadDetect(config: Config): Promise<GraphModel> {
   if (env.initial) models[0] = null;
@@ -135,10 +136,11 @@ async function detectParts(input: Tensor, config: Config, outputSize: [number, n
 
 export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
   const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0];
-  if ((skipped < (config.body.skipFrames || 0)) && config.skipFrame && cache !== null) {
+  if ((skipped < (config.body.skipFrames || 0)) && ((config.body.skipTime || 0) <= (now() - last)) && config.skipFrame && cache !== null) {
     skipped++;
   } else {
     cache = await detectParts(input, config, outputSize);
+    last = now();
     skipped = 0;
   }
   if (cache) return [cache];

src/body/efficientpose.ts

@@ -4,7 +4,7 @@
  * Based on: [**EfficientPose**](https://github.com/daniegr/EfficientPose)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './efficientposecoords';
 import type { BodyResult, Point } from '../result';
@@ -13,7 +13,7 @@ import type { Config } from '../config';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
-
+let last = 0;
 const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} };
 
 // const keypoints: Array<BodyKeypoint> = [];
@@ -50,12 +50,7 @@ function max2d(inputs, minScore) {
 }
 
 export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
-  /** blazepose caching
+  if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0 && ((config.body.skipTime || 0) <= (now() - last))) {
-   * not fully implemented
-   * 1. if skipFrame returned cached
-   * 2. run detection based on squared full frame
-   */
-  if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0) {
     skipped++;
     return [cache];
   }
@@ -71,6 +66,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
 
     let resT;
     if (config.body.enabled) resT = await model?.predict(tensor);
+    last = now();
     tf.dispose(tensor);
 
     if (resT) {

src/body/movenet.ts

@@ -4,7 +4,7 @@
  * Based on: [**MoveNet**](https://blog.tensorflow.org/2021/05/next-generation-pose-detection-with-movenet-and-tensorflowjs.html)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as box from '../util/box';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './movenetcoords';
@@ -23,9 +23,11 @@ let skipped = Number.MAX_SAFE_INTEGER;
 const cache: {
   boxes: Array<Box>, // unused
   bodies: Array<BodyResult>;
+  last: number,
 } = {
   boxes: [],
   bodies: [],
+  last: 0,
 };
 
 export async function load(config: Config): Promise<GraphModel> {
@@ -129,17 +131,10 @@ async function parseMultiPose(res, config, image, inputBox) {
 }
 
 export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
-  /** movenet caching
-   * 1. if skipFrame returned cached
-   * 2. if enough cached boxes run using cached boxes
-   * 3. if not enough detected bodies rerun using full frame
-   * 4. regenerate cached boxes based on current keypoints
-   */
-
   if (!model || !model?.inputs[0].shape) return []; // something is wrong with the model
   if (!config.skipFrame) cache.boxes.length = 0; // allowed to use cache or not
   skipped++; // increment skip frames
-  if (config.skipFrame && (skipped <= (config.body.skipFrames || 0))) {
+  if (config.skipFrame && (skipped <= (config.body.skipFrames || 0) && ((config.body.skipTime || 0) <= (now() - cache.last)))) {
     return cache.bodies; // return cached results without running anything
   }
   return new Promise(async (resolve) => {
@@ -181,6 +176,7 @@ export async function predict(input: Tensor, config: Config): Promise<BodyResult
     // run detection on squared input and no cached boxes
     t.input = fix.padInput(input, inputSize);
     t.res = await model?.predict(t.input) as Tensor;
+    cache.last = now();
     const res = await t.res.array();
     cache.bodies = (t.res.shape[2] === 17)
       ? await parseSinglePose(res, config, input, [0, 0, 1, 1])

src/config.ts

@@ -1,53 +1,42 @@
 /* eslint-disable indent */
 /* eslint-disable no-multi-spaces */
 
-/** Dectector part of face configuration */
+export interface GenericConfig {
-export interface FaceDetectorConfig {
+  enabled: boolean,
   modelPath: string,
+  skipFrames: number,
+  skipTime: number,
+}
+
+/** Dectector part of face configuration */
+export interface FaceDetectorConfig extends GenericConfig {
   rotation: boolean,
   maxDetected: number,
-  skipFrames: number,
   minConfidence: number,
   iouThreshold: number,
   return: boolean,
 }
 
 /** Mesh part of face configuration */
-export interface FaceMeshConfig {
+export type FaceMeshConfig = GenericConfig
-  enabled: boolean,
-  modelPath: string,
-}
 
 /** Iris part of face configuration */
-export interface FaceIrisConfig {
+export type FaceIrisConfig = GenericConfig
-  enabled: boolean,
-  modelPath: string,
-}
 
 /** Description or face embedding part of face configuration
  * - also used by age and gender detection
  */
-export interface FaceDescriptionConfig {
+export interface FaceDescriptionConfig extends GenericConfig {
-  enabled: boolean,
-  modelPath: string,
-  skipFrames: number,
   minConfidence: number,
 }
 
 /** Emotion part of face configuration */
-export interface FaceEmotionConfig {
+export interface FaceEmotionConfig extends GenericConfig {
-  enabled: boolean,
   minConfidence: number,
-  skipFrames: number,
-  modelPath: string,
 }
 
 /** Emotion part of face configuration */
-export interface FaceAntiSpoofConfig {
+export type FaceAntiSpoofConfig = GenericConfig
-  enabled: boolean,
-  skipFrames: number,
-  modelPath: string,
-}
 
 /** Controlls and configures all face-specific options:
  * - face detection, face mesh detection, age, gender, emotion detection and face description
@@ -86,18 +75,15 @@ export interface FaceConfig {
  * Changing `modelPath` will change module responsible for hand detection and tracking
  * Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json`
 */
-export interface BodyConfig {
+export interface BodyConfig extends GenericConfig {
-  enabled: boolean,
-  modelPath: string,
   maxDetected: number,
   minConfidence: number,
-  skipFrames: number,
   detector?: {
     modelPath: string
   },
 }
 
-/** Controlls and configures all hand detection specific options
+/** Controls and configures all hand detection specific options
  *
  * Parameters:
  * - enabled: true/false
@@ -113,10 +99,8 @@ export interface BodyConfig {
  * Changing `detector.modelPath` will change module responsible for hand detection and tracking
  * Allowed values are `handdetect.json` and `handtrack.json`
 */
-export interface HandConfig {
+export interface HandConfig extends GenericConfig {
-  enabled: boolean,
   rotation: boolean,
-  skipFrames: number,
   minConfidence: number,
   iouThreshold: number,
   maxDetected: number,
@@ -139,13 +123,10 @@ export interface HandConfig {
  * Changing `modelPath` will change module responsible for hand detection and tracking
  * Allowed values are `mb3-centernet.json` and `nanodet.json`
 */
-export interface ObjectConfig {
+export interface ObjectConfig extends GenericConfig {
-  enabled: boolean,
-  modelPath: string,
   minConfidence: number,
   iouThreshold: number,
   maxDetected: number,
-  skipFrames: number,
 }
 
 /** Controlls and configures all body segmentation module
@@ -368,9 +349,8 @@ const config: Config = {
                              // should be set to the minimum number for performance
       skipFrames: 11,        // how many max frames to go without re-running the face bounding box detector
                              // only used when cacheSensitivity is not zero
-                             // e.g., if model is running st 25 FPS, we can re-use existing bounding
+      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
-                             // box for updated face analysis as the head does not move fast
+                             // only used when cacheSensitivity is not zero
-                             // in short time (10 * 1/25 = 0.25 sec)
       minConfidence: 0.2,    // threshold for discarding a prediction
       iouThreshold: 0.1,     // ammount of overlap between two detected objects before one object is removed
       return: false,         // return extracted face as tensor
@@ -393,6 +373,8 @@ const config: Config = {
       minConfidence: 0.1,    // threshold for discarding a prediction
       skipFrames: 12,        // how max many frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
+      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+                             // only used when cacheSensitivity is not zero
       modelPath: 'emotion.json',  // face emotion model, can be absolute path or relative to modelBasePath
     },
 
@@ -403,6 +385,8 @@ const config: Config = {
                              // can be either absolute path or relative to modelBasePath
       skipFrames: 13,        // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
+      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+                             // only used when cacheSensitivity is not zero
       minConfidence: 0.1,    // threshold for discarding a prediction
     },
 
@@ -410,6 +394,8 @@ const config: Config = {
       enabled: false,
       skipFrames: 14,        // how max many frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
+      skipTime: 2000,        // how many ms to go without re-running the face bounding box detector
+                             // only used when cacheSensitivity is not zero
       modelPath: 'antispoof.json',  // face description model
                              // can be either absolute path or relative to modelBasePath
     },
@@ -429,6 +415,8 @@ const config: Config = {
     minConfidence: 0.3,      // threshold for discarding a prediction
     skipFrames: 1,           // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
+    skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
+                             // only used when cacheSensitivity is not zero
 },
 
   hand: {
@@ -438,9 +426,8 @@ const config: Config = {
                              // only valid for `handdetect` variation
     skipFrames: 2,           // how many max frames to go without re-running the hand bounding box detector
                              // only used when cacheSensitivity is not zero
-                             // e.g., if model is running st 25 FPS, we can re-use existing bounding
+    skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
-                             // box for updated hand skeleton analysis as the hand
+                             // only used when cacheSensitivity is not zero
-                             // hasn't moved much in short time (10 * 1/25 = 0.25 sec)
     minConfidence: 0.50,     // threshold for discarding a prediction
     iouThreshold: 0.2,       // ammount of overlap between two detected objects before one object is removed
     maxDetected: -1,         // maximum number of hands detected in the input
@@ -465,6 +452,8 @@ const config: Config = {
     maxDetected: 10,         // maximum number of objects detected in the input
     skipFrames: 15,          // how many max frames to go without re-running the detector
                              // only used when cacheSensitivity is not zero
+    skipTime: 2000,          // how many ms to go without re-running the face bounding box detector
+                             // only used when cacheSensitivity is not zero
   },
 
   segmentation: {

src/face/antispoof.ts

@@ -2,7 +2,7 @@
  * Anti-spoofing model implementation
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import * as tf from '../../dist/tfjs.esm.js';
@@ -12,6 +12,7 @@ let model: GraphModel | null;
 const cached: Array<number> = [];
 let skipped = Number.MAX_SAFE_INTEGER;
 let lastCount = 0;
+let last = 0;
 
 export async function load(config: Config): Promise<GraphModel> {
   if (env.initial) model = null;
@@ -25,7 +26,7 @@ export async function load(config: Config): Promise<GraphModel> {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.antispoof?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && cached[idx]) {
+  if ((skipped < (config.face.antispoof?.skipFrames || 0)) && ((config.face.antispoof?.skipTime || 0) <= (now() - last)) && config.skipFrame && (lastCount === count) && cached[idx]) {
     skipped++;
     return cached[idx];
   }
@@ -36,6 +37,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
     const num = (await res.data())[0];
     cached[idx] = Math.round(100 * num) / 100;
     lastCount = count;
+    last = now();
     tf.dispose([resize, res]);
     resolve(cached[idx]);
   });

src/face/facemesh.ts

@@ -7,7 +7,7 @@
  * - Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as blazeface from './blazeface';
 import * as util from './facemeshutil';
@@ -23,11 +23,14 @@ let boxCache: Array<BoxCache> = [];
 let model: GraphModel | null = null;
 let inputSize = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
+let lastTime = 0;
 let detectedFaces = 0;
 
 export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
-  if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0))) { // reset cached boxes
+  // reset cached boxes
+  if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)))) {
     const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
+    lastTime = now();
     boxCache = []; // empty cache
     for (const possible of newBoxes.boxes) { // extract data from detector
       const startPoint = await possible.box.startPoint.data() as unknown as Point;

src/face/faceres.ts

@@ -7,7 +7,7 @@
  * Based on: [**HSE-FaceRes**](https://github.com/HSE-asavchenko/HSE_FaceRec_tf)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import type { Config } from '../config';
@@ -21,6 +21,7 @@ const last: Array<{
   descriptor: number[],
 }> = [];
 
+let lastTime = 0;
 let lastCount = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
@@ -90,15 +91,12 @@ export function enhance(input): Tensor {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.description?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
+  if ((skipped < (config.face.description?.skipFrames || 0)) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
     skipped++;
     return last[idx];
   }
   skipped = 0;
   return new Promise(async (resolve) => {
-    const enhanced = enhance(image);
-
-    let resT;
     const obj = {
       age: <number>0,
       gender: <string>'unknown',
@@ -106,11 +104,13 @@ export async function predict(image: Tensor, config: Config, idx, count) {
       descriptor: <number[]>[],
     };
 
-    if (config.face.description?.enabled) resT = await model?.predict(enhanced);
+    if (config.face.description?.enabled) {
-    tf.dispose(enhanced);
+      const enhanced = enhance(image);
-
+      const resT = await model?.predict(enhanced) as Tensor[];
-    if (resT) {
+      lastTime = now();
-      const gender = await resT.find((t) => t.shape[1] === 1).data();
+      tf.dispose(enhanced);
+      const genderT = await resT.find((t) => t.shape[1] === 1) as Tensor;
+      const gender = await genderT.data();
       const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
       if (confidence > (config.face.description?.minConfidence || 0)) {
         obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
@@ -119,15 +119,16 @@ export async function predict(image: Tensor, config: Config, idx, count) {
       const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1);
       const age = (await argmax.data())[0];
       tf.dispose(argmax);
-      const all = await resT.find((t) => t.shape[1] === 100).data();
+      const ageT = resT.find((t) => t.shape[1] === 100) as Tensor;
+      const all = await ageT.data();
       obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
 
       const desc = resT.find((t) => t.shape[1] === 1024);
       // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
       // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
-
+      const descriptor = desc ? await desc.data() : <number[]>[];
-      const descriptor = await desc.data();
+      // obj.descriptor = [...descriptor];
-      obj.descriptor = [...descriptor];
+      obj.descriptor = Array.from(descriptor);
       resT.forEach((t) => tf.dispose(t));
     }
     last[idx] = obj;

src/gear/emotion.ts

@@ -4,7 +4,7 @@
  * [**Oarriaga**](https://github.com/oarriaga/face_classification)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import * as tf from '../../dist/tfjs.esm.js';
@@ -15,6 +15,7 @@ let model: GraphModel | null;
 // let last: Array<{ score: number, emotion: string }> = [];
 const last: Array<Array<{ score: number, emotion: string }>> = [];
 let lastCount = 0;
+let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
 // tuning values
@@ -32,39 +33,40 @@ export async function load(config: Config): Promise<GraphModel> {
 
 export async function predict(image: Tensor, config: Config, idx, count) {
   if (!model) return null;
-  if ((skipped < (config.face.emotion?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
+  if ((skipped < (config.face.emotion?.skipFrames || 0)) && ((config.face.emotion?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
     skipped++;
     return last[idx];
   }
   skipped = 0;
   return new Promise(async (resolve) => {
-    const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false);
-    const [red, green, blue] = tf.split(resize, 3, 3);
-    tf.dispose(resize);
-    // weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
-    const redNorm = tf.mul(red, rgb[0]);
-    const greenNorm = tf.mul(green, rgb[1]);
-    const blueNorm = tf.mul(blue, rgb[2]);
-    tf.dispose(red);
-    tf.dispose(green);
-    tf.dispose(blue);
-    const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
-    tf.dispose(redNorm);
-    tf.dispose(greenNorm);
-    tf.dispose(blueNorm);
-    const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
-    tf.dispose(grayscale);
     const obj: Array<{ score: number, emotion: string }> = [];
     if (config.face.emotion?.enabled) {
+      const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false);
+      const [red, green, blue] = tf.split(resize, 3, 3);
+      tf.dispose(resize);
+      // weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
+      const redNorm = tf.mul(red, rgb[0]);
+      const greenNorm = tf.mul(green, rgb[1]);
+      const blueNorm = tf.mul(blue, rgb[2]);
+      tf.dispose(red);
+      tf.dispose(green);
+      tf.dispose(blue);
+      const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
+      tf.dispose(redNorm);
+      tf.dispose(greenNorm);
+      tf.dispose(blueNorm);
+      const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
+      tf.dispose(grayscale);
       const emotionT = await model?.predict(normalize) as Tensor; // result is already in range 0..1, no need for additional activation
+      lastTime = now();
       const data = await emotionT.data();
       tf.dispose(emotionT);
       for (let i = 0; i < data.length; i++) {
         if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
       }
       obj.sort((a, b) => b.score - a.score);
+      tf.dispose(normalize);
     }
-    tf.dispose(normalize);
     last[idx] = obj;
     lastCount = count;
     resolve(obj);

src/gear/ssrnet-age.ts

@@ -6,15 +6,15 @@
  * Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
-
 let last = { age: 0 };
+let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -33,7 +33,7 @@ export async function load(config: Config | any) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function predict(image: Tensor, config: Config | any) {
   if (!model) return null;
-  if ((skipped < config.face.age.skipFrames) && config.skipFrame && last.age && (last.age > 0)) {
+  if ((skipped < config.face.age.skipFrames) && ((config.face.age.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.age && (last.age > 0)) {
     skipped++;
     return last;
   }
@@ -48,6 +48,7 @@ export async function predict(image: Tensor, config: Config | any) {
     const obj = { age: 0 };
 
     if (config.face.age.enabled) ageT = await model.predict(enhance);
+    lastTime = now();
     tf.dispose(enhance);
 
     if (ageT) {

src/gear/ssrnet-gender.ts

@@ -6,7 +6,7 @@
  * Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
@@ -14,6 +14,7 @@ import { env } from '../util/env';
 
 let model: GraphModel | null;
 let last = { gender: '' };
+let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 let alternative = false;
 
@@ -35,7 +36,7 @@ export async function load(config: Config | any) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function predict(image: Tensor, config: Config | any) {
   if (!model) return null;
-  if ((skipped < config.face.gender.skipFrames) && config.skipFrame && last.gender !== '') {
+  if ((skipped < config.face.gender.skipFrames) && ((config.face.gender.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.gender !== '') {
     skipped++;
     return last;
   }
@@ -63,6 +64,7 @@ export async function predict(image: Tensor, config: Config | any) {
     const obj = { gender: '', confidence: 0 };
 
     if (config.face.gender.enabled) genderT = await model.predict(enhance);
+    lastTime = now();
     tf.dispose(enhance);
 
     if (genderT) {

src/hand/handposepipeline.ts

@@ -8,12 +8,14 @@ import * as util from './handposeutil';
 import type * as detector from './handposedetector';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import { env } from '../util/env';
+import { now } from '../util/util';
 
 const palmBoxEnlargeFactor = 5; // default 3
 const handBoxEnlargeFactor = 1.65; // default 1.65
 const palmLandmarkIds = [0, 5, 9, 13, 17, 1, 2];
 const palmLandmarksPalmBase = 0;
 const palmLandmarksMiddleFingerBase = 2;
+let lastTime = 0;
 
 export class HandPipeline {
   handDetector: detector.HandDetector;
@@ -90,7 +92,7 @@ export class HandPipeline {
     let boxes;
 
     // console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
-    if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) {
+    if ((this.skipped === 0) || ((this.skipped > config.hand.skipFrames) && ((config.hand.skipTime || 0) <= (now() - lastTime))) || !config.hand.landmarks || !config.skipFrame) {
       boxes = await this.handDetector.estimateHandBounds(image, config);
       this.skipped = 0;
     }
@@ -121,6 +123,7 @@ export class HandPipeline {
         tf.dispose(croppedInput);
         tf.dispose(rotatedImage);
         const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>;
+        lastTime = now();
         tf.dispose(handImage);
         const confidence = (await confidenceT.data())[0];
         tf.dispose(confidenceT);

src/hand/handtrack.ts

@@ -6,7 +6,7 @@
  * - Hand Tracking: [**HandTracking**](https://github.com/victordibia/handtracking)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as box from '../util/box';
 import * as tf from '../../dist/tfjs.esm.js';
 import type { HandResult, Box, Point } from '../result';
@@ -29,6 +29,7 @@ const maxDetectorResolution = 512;
 const detectorExpandFact = 1.4;
 
 let skipped = 0;
+let lastTime = 0;
 let outputSize: [number, number] = [0, 0];
 
 type HandDetectResult = {
@@ -184,7 +185,7 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
 export async function predict(input: Tensor, config: Config): Promise<HandResult[]> {
   /** handtrack caching
    * 1. if skipFrame returned cached
-   * 2. if any cached results but although not sure if its enough we continute anyhow for 5x skipframes
+   * 2. if any cached results but although not sure if its enough we continute anyhow for 3x skipframes
    * 3. if not skipframe or eventually rerun detector to generated new cached boxes and reset skipped
    * 4. generate cached boxes based on detected keypoints
    */
@@ -192,16 +193,17 @@ export async function predict(input: Tensor, config: Config): Promise<HandResult
   outputSize = [input.shape[2] || 0, input.shape[1] || 0];
 
   skipped++; // increment skip frames
-  if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0))) {
+  if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0)) && ((config.hand.skipTime || 0) <= (now() - lastTime))) {
     return cache.hands; // return cached results without running anything
   }
   return new Promise(async (resolve) => {
     if (config.skipFrame && cache.hands.length === config.hand.maxDetected) { // we have all detected hands
       cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
-    } else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && cache.hands.length > 0) { // we have some cached results but although not sure if its enough we continute anyhow for bit longer
+    } else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && ((config.hand.skipTime || 0) <= 3 * (now() - lastTime)) && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer
       cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
     } else { // finally rerun detector
       cache.boxes = await detectHands(input, config);
+      lastTime = now();
       cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
       skipped = 0;
     }

src/object/centernet.ts

@@ -4,7 +4,7 @@
  * Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { labels } from './labels';
 import type { ObjectResult, Box } from '../result';
@@ -16,6 +16,7 @@ import { fakeOps } from '../tfjs/backend';
 let model: GraphModel | null;
 let inputSize = 0;
 let last: ObjectResult[] = [];
+let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
 export async function load(config: Config): Promise<GraphModel> {
@@ -78,7 +79,7 @@ async function process(res: Tensor | null, outputShape, config: Config) {
 }
 
 export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> {
-  if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) {
+  if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
     skipped++;
     return last;
   }
@@ -88,6 +89,7 @@ export async function predict(input: Tensor, config: Config): Promise<ObjectResu
     const outputSize = [input.shape[2], input.shape[1]];
     const resize = tf.image.resizeBilinear(input, [inputSize, inputSize]);
     const objectT = config.object.enabled ? model?.execute(resize, ['tower_0/detections']) as Tensor : null;
+    lastTime = now();
     tf.dispose(resize);
 
     const obj = await process(objectT, outputSize, config);

src/object/nanodet.ts

@@ -4,7 +4,7 @@
  * Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet)
  */
 
-import { log, join } from '../util/util';
+import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { labels } from './labels';
 import type { ObjectResult, Box } from '../result';
@@ -14,6 +14,7 @@ import { env } from '../util/env';
 
 let model;
 let last: Array<ObjectResult> = [];
+let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
 const scaleBox = 2.5; // increase box size
@@ -106,7 +107,7 @@ async function process(res, inputSize, outputShape, config) {
 }
 
 export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> {
-  if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) {
+  if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
     skipped++;
     return last;
   }
@@ -122,6 +123,7 @@ export async function predict(image: Tensor, config: Config): Promise<ObjectResu
 
     let objectT;
     if (config.object.enabled) objectT = await model.predict(transpose);
+    lastTime = now();
     tf.dispose(transpose);
 
     const obj = await process(objectT, model.inputSize, outputSize, config);