caching improvements

CHANGELOG.md

@@ -9,11 +9,15 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **human 1.9.0 beta with breaking changes regarding caching** 2021/05/18 mandic00@live.com
-
-
-### **origin/main** 2021/05/18 mandic00@live.com
+### **HEAD -> main** 2021/05/19 mandic00@live.com
 
+- sanitize server input
+- remove nanodet weights from default distribution
+- add experimental mb3-centernet object detection
+- individual model skipframes values still max high threshold for caching
+- config.videooptimized has been removed and config.cachesensitivity has been added instead
+- caching determination is now dynamic based on detection of input change and not based on input types
+- human 1.9.0 beta with breaking changes regarding caching
 
 ### **1.8.5** 2021/05/18 mandic00@live.com
 

TODO.md

@@ -6,7 +6,7 @@ N/A
 
 ## Exploring Features
 
-N/A
+- Output interpolation for draw
 
 ## Explore Models
 

demo/index.js

@@ -9,7 +9,7 @@ import webRTC from './helpers/webrtc.js';
 let human;
 
 const userConfig = {
-  warmup: 'full',
+  warmup: 'none',
   /*
   backend: 'webgl',
   async: false,
@@ -54,7 +54,7 @@ const ui = {
   camera: {}, // internal, holds details of webcam details
   detectFPS: [], // internal, holds fps values for detection performance
   drawFPS: [], // internal, holds fps values for draw performance
-  buffered: true, // should output be buffered between frames
+  buffered: false, // should output be buffered between frames
   drawWarmup: false, // debug only, should warmup image processing be displayed on startup
   drawThread: null, // internl, perform draw operations in a separate thread
   detectThread: null, // internl, perform detect operations in a separate thread

src/config.ts

@@ -201,7 +201,7 @@ const config: Config = {
                              // warmup pre-initializes all models for faster inference but can take
                              // significant time on startup
                              // only used for `webgl` and `humangl` backends
-  cacheSensitivity: 0.005,   // cache sensitivity
+  cacheSensitivity: 0.01,    // cache sensitivity
                              // values 0..1 where 0.01 means reset cache if input changed more than 1%
                              // set to 0 to disable caching
   filter: {                  // run input through image filters before inference

src/human.ts

@@ -119,7 +119,8 @@ export class Human {
   #analyzeMemoryLeaks: boolean;
   #checkSanity: boolean;
   #firstRun: boolean;
-  #lastInputSum: number
+  #lastInputSum: number;
+  #lastCacheDiff: number;
 
   // definition end
 
@@ -137,6 +138,7 @@ export class Human {
     this.#analyzeMemoryLeaks = false;
     this.#checkSanity = false;
     this.#firstRun = true;
+    this.#lastCacheDiff = 0;
     this.perf = {};
     // object that contains all initialized models
     this.models = {
@@ -335,6 +337,8 @@ export class Human {
       // this.tf.enableDebugMode();
       if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
         this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
+        this.tf.ENV.set('WEBGL_CPU_FORWARD', true);
+        tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
         this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
         if (typeof this.config['deallocate'] !== 'undefined') {
           log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
@@ -352,15 +356,27 @@ export class Human {
   /** @hidden */
   #skipFrame = async (input) => {
     if (this.config.cacheSensitivity === 0) return false;
-    const resizeFact = 50;
+    const resizeFact = 40;
     const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
+    // use tensor sum
     const sumT = this.tf.sum(reduced);
-    reduced.dispose();
     const sum = sumT.dataSync()[0] as number;
     sumT.dispose();
+    // use js loop sum
+    /*
+    const reducedData = reduced.dataSync();
+    let sum = 0;
+    for (let i = 0; i < reducedData.length; i++) sum += reducedData[i];
+    */
+    reduced.dispose();
     const diff = Math.max(sum, this.#lastInputSum) / Math.min(sum, this.#lastInputSum) - 1;
     this.#lastInputSum = sum;
-    return diff < this.config.cacheSensitivity;
+    // if previous frame was skipped, skip this frame if changed more than cacheSensitivity
+    // if previous frame was not skipped, then look for cacheSensitivity or difference larger than one in previous frame to avoid resetting cache in subsequent frames unnecessarily
+    const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff);
+    // if difference is above 4x threshold, don't use last value to force reset cache for significant change of scenes or images
+    this.#lastCacheDiff = diff > 4 * this.config.cacheSensitivity ? 0 : diff;
+    return skipFrame;
   }
 
   /** Main detection method