From 6ced1341570f92182930d3b439d9e7761b5e9211 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 20 May 2021 19:14:07 -0400 Subject: [PATCH] caching improvements --- CHANGELOG.md | 12 ++++++++---- TODO.md | 2 +- demo/index.js | 4 ++-- src/config.ts | 2 +- src/human.ts | 24 ++++++++++++++++++++---- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d786897..f87daaee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,15 @@ Repository: **** ## Changelog -### **human 1.9.0 beta with breaking changes regarding caching** 2021/05/18 mandic00@live.com - - -### **origin/main** 2021/05/18 mandic00@live.com +### **HEAD -> main** 2021/05/19 mandic00@live.com +- sanitize server input +- remove nanodet weights from default distribution +- add experimental mb3-centernet object detection +- individual model skipframes values still max high threshold for caching +- config.videooptimized has been removed and config.cachesensitivity has been added instead +- caching determination is now dynamic based on detection of input change and not based on input types +- human 1.9.0 beta with breaking changes regarding caching ### **1.8.5** 2021/05/18 mandic00@live.com diff --git a/TODO.md b/TODO.md index 4ed2df4a..0257d05c 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,7 @@ N/A ## Exploring Features -N/A +- Output interpolation for draw ## Explore Models diff --git a/demo/index.js b/demo/index.js index f3039707..858084a9 100644 --- a/demo/index.js +++ b/demo/index.js @@ -9,7 +9,7 @@ import webRTC from './helpers/webrtc.js'; let human; const userConfig = { - warmup: 'full', + warmup: 'none', /* backend: 'webgl', async: false, @@ -54,7 +54,7 @@ const ui = { camera: {}, // internal, holds details of webcam details detectFPS: [], // internal, holds fps values for detection performance drawFPS: [], // internal, holds fps values for draw performance - buffered: true, // should output be buffered between frames + buffered: false, // should output be buffered between frames drawWarmup: false, // debug only, should warmup image processing be displayed on startup drawThread: null, // internl, perform draw operations in a separate thread detectThread: null, // internl, perform detect operations in a separate thread diff --git a/src/config.ts b/src/config.ts index d2f64c26..6bc598ce 100644 --- a/src/config.ts +++ b/src/config.ts @@ -201,7 +201,7 @@ const config: Config = { // warmup pre-initializes all models for faster inference but can take // significant time on startup // only used for `webgl` and `humangl` backends - cacheSensitivity: 0.005, // cache sensitivity + cacheSensitivity: 0.01, // cache sensitivity // values 0..1 where 0.01 means reset cache if input changed more than 1% // set to 0 to disable caching filter: { // run input through image filters before inference diff --git a/src/human.ts b/src/human.ts index ad1b0277..48cb64ad 100644 --- a/src/human.ts +++ b/src/human.ts @@ -119,7 +119,8 @@ export class Human { #analyzeMemoryLeaks: boolean; #checkSanity: boolean; #firstRun: boolean; - #lastInputSum: number + #lastInputSum: number; + #lastCacheDiff: number; // definition end @@ -137,6 +138,7 @@ export class Human { this.#analyzeMemoryLeaks = false; this.#checkSanity = false; this.#firstRun = true; + this.#lastCacheDiff = 0; this.perf = {}; // object that contains all initialized models this.models = { @@ -335,6 +337,8 @@ export class Human { // this.tf.enableDebugMode(); if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') { this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false); + this.tf.ENV.set('WEBGL_CPU_FORWARD', true); + tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true); if (typeof this.config['deallocate'] !== 'undefined') { log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true); @@ -352,15 +356,27 @@ export class Human { /** @hidden */ #skipFrame = async (input) => { if (this.config.cacheSensitivity === 0) return false; - const resizeFact = 50; + const resizeFact = 40; const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]); + // use tensor sum const sumT = this.tf.sum(reduced); - reduced.dispose(); const sum = sumT.dataSync()[0] as number; sumT.dispose(); + // use js loop sum + /* + const reducedData = reduced.dataSync(); + let sum = 0; + for (let i = 0; i < reducedData.length; i++) sum += reducedData[i]; + */ + reduced.dispose(); const diff = Math.max(sum, this.#lastInputSum) / Math.min(sum, this.#lastInputSum) - 1; this.#lastInputSum = sum; - return diff < this.config.cacheSensitivity; + // if previous frame was skipped, skip this frame if changed more than cacheSensitivity + // if previous frame was not skipped, then look for cacheSensitivity or difference larger than one in previous frame to avoid resetting cache in subsequent frames unnecessarily + const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff); + // if difference is above 4x threshold, don't use last value to force reset cache for significant change of scenes or images + this.#lastCacheDiff = diff > 4 * this.config.cacheSensitivity ? 0 : diff; + return skipFrame; } /** Main detection method