mirror of https://github.com/vladmandic/human
caching improvements
parent
a5b5352ea6
commit
1c52d42e24
12
CHANGELOG.md
12
CHANGELOG.md
|
@ -9,11 +9,15 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
||||||
|
|
||||||
## Changelog
|
## Changelog
|
||||||
|
|
||||||
### **human 1.9.0 beta with breaking changes regarding caching** 2021/05/18 mandic00@live.com
|
### **HEAD -> main** 2021/05/19 mandic00@live.com
|
||||||
|
|
||||||
|
|
||||||
### **origin/main** 2021/05/18 mandic00@live.com
|
|
||||||
|
|
||||||
|
- sanitize server input
|
||||||
|
- remove nanodet weights from default distribution
|
||||||
|
- add experimental mb3-centernet object detection
|
||||||
|
- individual model skipframes values still max high threshold for caching
|
||||||
|
- config.videooptimized has been removed and config.cachesensitivity has been added instead
|
||||||
|
- caching determination is now dynamic based on detection of input change and not based on input types
|
||||||
|
- human 1.9.0 beta with breaking changes regarding caching
|
||||||
|
|
||||||
### **1.8.5** 2021/05/18 mandic00@live.com
|
### **1.8.5** 2021/05/18 mandic00@live.com
|
||||||
|
|
||||||
|
|
2
TODO.md
2
TODO.md
|
@ -6,7 +6,7 @@ N/A
|
||||||
|
|
||||||
## Exploring Features
|
## Exploring Features
|
||||||
|
|
||||||
N/A
|
- Output interpolation for draw
|
||||||
|
|
||||||
## Explore Models
|
## Explore Models
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,7 @@ import webRTC from './helpers/webrtc.js';
|
||||||
let human;
|
let human;
|
||||||
|
|
||||||
const userConfig = {
|
const userConfig = {
|
||||||
warmup: 'full',
|
warmup: 'none',
|
||||||
/*
|
/*
|
||||||
backend: 'webgl',
|
backend: 'webgl',
|
||||||
async: false,
|
async: false,
|
||||||
|
@ -54,7 +54,7 @@ const ui = {
|
||||||
camera: {}, // internal, holds details of webcam details
|
camera: {}, // internal, holds details of webcam details
|
||||||
detectFPS: [], // internal, holds fps values for detection performance
|
detectFPS: [], // internal, holds fps values for detection performance
|
||||||
drawFPS: [], // internal, holds fps values for draw performance
|
drawFPS: [], // internal, holds fps values for draw performance
|
||||||
buffered: true, // should output be buffered between frames
|
buffered: false, // should output be buffered between frames
|
||||||
drawWarmup: false, // debug only, should warmup image processing be displayed on startup
|
drawWarmup: false, // debug only, should warmup image processing be displayed on startup
|
||||||
drawThread: null, // internl, perform draw operations in a separate thread
|
drawThread: null, // internl, perform draw operations in a separate thread
|
||||||
detectThread: null, // internl, perform detect operations in a separate thread
|
detectThread: null, // internl, perform detect operations in a separate thread
|
||||||
|
|
|
@ -201,7 +201,7 @@ const config: Config = {
|
||||||
// warmup pre-initializes all models for faster inference but can take
|
// warmup pre-initializes all models for faster inference but can take
|
||||||
// significant time on startup
|
// significant time on startup
|
||||||
// only used for `webgl` and `humangl` backends
|
// only used for `webgl` and `humangl` backends
|
||||||
cacheSensitivity: 0.005, // cache sensitivity
|
cacheSensitivity: 0.01, // cache sensitivity
|
||||||
// values 0..1 where 0.01 means reset cache if input changed more than 1%
|
// values 0..1 where 0.01 means reset cache if input changed more than 1%
|
||||||
// set to 0 to disable caching
|
// set to 0 to disable caching
|
||||||
filter: { // run input through image filters before inference
|
filter: { // run input through image filters before inference
|
||||||
|
|
24
src/human.ts
24
src/human.ts
|
@ -119,7 +119,8 @@ export class Human {
|
||||||
#analyzeMemoryLeaks: boolean;
|
#analyzeMemoryLeaks: boolean;
|
||||||
#checkSanity: boolean;
|
#checkSanity: boolean;
|
||||||
#firstRun: boolean;
|
#firstRun: boolean;
|
||||||
#lastInputSum: number
|
#lastInputSum: number;
|
||||||
|
#lastCacheDiff: number;
|
||||||
|
|
||||||
// definition end
|
// definition end
|
||||||
|
|
||||||
|
@ -137,6 +138,7 @@ export class Human {
|
||||||
this.#analyzeMemoryLeaks = false;
|
this.#analyzeMemoryLeaks = false;
|
||||||
this.#checkSanity = false;
|
this.#checkSanity = false;
|
||||||
this.#firstRun = true;
|
this.#firstRun = true;
|
||||||
|
this.#lastCacheDiff = 0;
|
||||||
this.perf = {};
|
this.perf = {};
|
||||||
// object that contains all initialized models
|
// object that contains all initialized models
|
||||||
this.models = {
|
this.models = {
|
||||||
|
@ -335,6 +337,8 @@ export class Human {
|
||||||
// this.tf.enableDebugMode();
|
// this.tf.enableDebugMode();
|
||||||
if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
|
if (this.tf.getBackend() === 'webgl' || this.tf.getBackend() === 'humangl') {
|
||||||
this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
|
this.tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
|
||||||
|
this.tf.ENV.set('WEBGL_CPU_FORWARD', true);
|
||||||
|
tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
|
||||||
this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
|
this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
|
||||||
if (typeof this.config['deallocate'] !== 'undefined') {
|
if (typeof this.config['deallocate'] !== 'undefined') {
|
||||||
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
|
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
|
||||||
|
@ -352,15 +356,27 @@ export class Human {
|
||||||
/** @hidden */
|
/** @hidden */
|
||||||
#skipFrame = async (input) => {
|
#skipFrame = async (input) => {
|
||||||
if (this.config.cacheSensitivity === 0) return false;
|
if (this.config.cacheSensitivity === 0) return false;
|
||||||
const resizeFact = 50;
|
const resizeFact = 40;
|
||||||
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
const reduced = input.resizeBilinear([Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
|
||||||
|
// use tensor sum
|
||||||
const sumT = this.tf.sum(reduced);
|
const sumT = this.tf.sum(reduced);
|
||||||
reduced.dispose();
|
|
||||||
const sum = sumT.dataSync()[0] as number;
|
const sum = sumT.dataSync()[0] as number;
|
||||||
sumT.dispose();
|
sumT.dispose();
|
||||||
|
// use js loop sum
|
||||||
|
/*
|
||||||
|
const reducedData = reduced.dataSync();
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < reducedData.length; i++) sum += reducedData[i];
|
||||||
|
*/
|
||||||
|
reduced.dispose();
|
||||||
const diff = Math.max(sum, this.#lastInputSum) / Math.min(sum, this.#lastInputSum) - 1;
|
const diff = Math.max(sum, this.#lastInputSum) / Math.min(sum, this.#lastInputSum) - 1;
|
||||||
this.#lastInputSum = sum;
|
this.#lastInputSum = sum;
|
||||||
return diff < this.config.cacheSensitivity;
|
// if previous frame was skipped, skip this frame if changed more than cacheSensitivity
|
||||||
|
// if previous frame was not skipped, then look for cacheSensitivity or difference larger than one in previous frame to avoid resetting cache in subsequent frames unnecessarily
|
||||||
|
const skipFrame = diff < Math.max(this.config.cacheSensitivity, this.#lastCacheDiff);
|
||||||
|
// if difference is above 4x threshold, don't use last value to force reset cache for significant change of scenes or images
|
||||||
|
this.#lastCacheDiff = diff > 4 * this.config.cacheSensitivity ? 0 : diff;
|
||||||
|
return skipFrame;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Main detection method
|
/** Main detection method
|
||||||
|
|
Loading…
Reference in New Issue