added custom webgl backend

pull/50/head
Vladimir Mandic 2020-12-12 18:34:30 -05:00
parent 9f7ece0b35
commit 977f92fa96
24 changed files with 6784 additions and 5280 deletions

View File

@ -3,7 +3,7 @@
export default { export default {
backend: 'webgl', // select tfjs backend to use backend: 'webgl', // select tfjs backend to use
wasmPath: '../assets/', // path for wasm binaries wasmPath: '../assets/', // path for wasm binaries
// only used for backend: wasm // only used for backend: wasm
async: true, // execute enabled models in parallel async: true, // execute enabled models in parallel
// this disables per-model performance data but // this disables per-model performance data but
@ -26,7 +26,7 @@ export default {
// must be disabled for images // must be disabled for images
// basically this skips object box boundary detection for every n frames // basically this skips object box boundary detection for every n frames
// while maintaining in-box detection since objects cannot move that fast // while maintaining in-box detection since objects cannot move that fast
warmup: 'full', // what to use for human.warmup(), can be 'none', 'face', 'full' warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full'
// warmup pre-initializes all models for faster inference but can take // warmup pre-initializes all models for faster inference but can take
// significant time on startup // significant time on startup
filter: { filter: {

View File

@ -4,13 +4,6 @@ import Menu from './menu.js';
import GLBench from './gl-bench.js'; import GLBench from './gl-bench.js';
const userConfig = {}; // add any user configuration overrides const userConfig = {}; // add any user configuration overrides
/*
const userConfig = {
async: false,
face: { enabled: false },
body: { enabled: false },
};
*/
const human = new Human(userConfig); const human = new Human(userConfig);
@ -441,7 +434,7 @@ function setupMenu() {
menu.image.addBool('polaroid', human.config.filter, 'polaroid'); menu.image.addBool('polaroid', human.config.filter, 'polaroid');
menu.process = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[2] }); menu.process = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[2] });
menu.process.addList('backend', ['cpu', 'webgl', 'wasm'], human.config.backend, (val) => human.config.backend = val); menu.process.addList('backend', ['cpu', 'webgl', 'wasm', 'humangl'], human.config.backend, (val) => human.config.backend = val);
menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val); menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val);
menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val); menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val);
menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val); menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val);

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1797523, "bytes": 1836469,
"imports": [] "imports": []
}, },
"demo/draw.js": { "demo/draw.js": {
@ -17,7 +17,7 @@
"imports": [] "imports": []
}, },
"demo/browser.js": { "demo/browser.js": {
"bytes": 25428, "bytes": 25337,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js" "path": "dist/human.esm.js"
@ -38,14 +38,14 @@
"dist/demo-browser-index.js.map": { "dist/demo-browser-index.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 2716716 "bytes": 2199626
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1790283 "bytesInOutput": 1829229
}, },
"demo/draw.js": { "demo/draw.js": {
"bytesInOutput": 7816 "bytesInOutput": 7816
@ -57,10 +57,10 @@
"bytesInOutput": 7382 "bytesInOutput": 7382
}, },
"demo/browser.js": { "demo/browser.js": {
"bytesInOutput": 19529 "bytesInOutput": 19539
} }
}, },
"bytes": 1844199 "bytes": 1883155
} }
} }
} }

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

422
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

52
dist/human.esm.json vendored
View File

@ -5,9 +5,17 @@
"imports": [] "imports": []
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1489546, "bytes": 1526803,
"imports": [] "imports": []
}, },
"src/tfjs/backend.js": {
"bytes": 1340,
"imports": [
{
"path": "dist/tfjs.esm.js"
}
]
},
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytes": 6986, "bytes": 6986,
"imports": [ "imports": [
@ -342,7 +350,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 9243, "bytes": 9241,
"imports": [] "imports": []
}, },
"src/sample.js": { "src/sample.js": {
@ -354,7 +362,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 16894, "bytes": 17068,
"imports": [ "imports": [
{ {
"path": "src/log.js" "path": "src/log.js"
@ -362,6 +370,9 @@
{ {
"path": "dist/tfjs.esm.js" "path": "dist/tfjs.esm.js"
}, },
{
"path": "src/tfjs/backend.js"
},
{ {
"path": "src/face/facemesh.js" "path": "src/face/facemesh.js"
}, },
@ -408,7 +419,7 @@
"dist/human.esm.js.map": { "dist/human.esm.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 2623471 "bytes": 2106327
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -417,7 +428,7 @@
], ],
"inputs": { "inputs": {
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytesInOutput": 5096 "bytesInOutput": 5106
}, },
"src/face/box.js": { "src/face/box.js": {
"bytesInOutput": 1578 "bytesInOutput": 1578
@ -429,25 +440,25 @@
"bytesInOutput": 30817 "bytesInOutput": 30817
}, },
"src/face/facepipeline.js": { "src/face/facepipeline.js": {
"bytesInOutput": 9314 "bytesInOutput": 9329
}, },
"src/face/facemesh.js": { "src/face/facemesh.js": {
"bytesInOutput": 2182 "bytesInOutput": 2202
}, },
"src/profile.js": { "src/profile.js": {
"bytesInOutput": 846 "bytesInOutput": 846
}, },
"src/age/age.js": { "src/age/age.js": {
"bytesInOutput": 1169 "bytesInOutput": 1181
}, },
"src/gender/gender.js": { "src/gender/gender.js": {
"bytesInOutput": 1915 "bytesInOutput": 1929
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1795 "bytesInOutput": 1807
}, },
"src/embedding/embedding.js": { "src/embedding/embedding.js": {
"bytesInOutput": 1310 "bytesInOutput": 1318
}, },
"src/body/modelBase.js": { "src/body/modelBase.js": {
"bytesInOutput": 615 "bytesInOutput": 615
@ -477,22 +488,22 @@
"bytesInOutput": 1913 "bytesInOutput": 1913
}, },
"src/body/modelPoseNet.js": { "src/body/modelPoseNet.js": {
"bytesInOutput": 1558 "bytesInOutput": 1569
}, },
"src/body/posenet.js": { "src/body/posenet.js": {
"bytesInOutput": 832 "bytesInOutput": 832
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 2731 "bytesInOutput": 2742
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 4447 "bytesInOutput": 4459
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127032 "bytesInOutput": 127032
}, },
"src/hand/handpose.js": { "src/hand/handpose.js": {
"bytesInOutput": 2007 "bytesInOutput": 2022
}, },
"src/gesture/gesture.js": { "src/gesture/gesture.js": {
"bytesInOutput": 2463 "bytesInOutput": 2463
@ -501,16 +512,19 @@
"bytesInOutput": 13576 "bytesInOutput": 13576
}, },
"src/image.js": { "src/image.js": {
"bytesInOutput": 4009 "bytesInOutput": 4041
}, },
"src/log.js": { "src/log.js": {
"bytesInOutput": 266 "bytesInOutput": 266
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1487547 "bytesInOutput": 1525301
},
"src/tfjs/backend.js": {
"bytesInOutput": 938
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 11073 "bytesInOutput": 11155
}, },
"src/hand/box.js": { "src/hand/box.js": {
"bytesInOutput": 1473 "bytesInOutput": 1473
@ -528,7 +542,7 @@
"bytesInOutput": 21 "bytesInOutput": 21
} }
}, },
"bytes": 1797523 "bytes": 1836469
} }
} }
} }

422
dist/human.js vendored

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

52
dist/human.json vendored
View File

@ -5,9 +5,17 @@
"imports": [] "imports": []
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1489546, "bytes": 1526803,
"imports": [] "imports": []
}, },
"src/tfjs/backend.js": {
"bytes": 1340,
"imports": [
{
"path": "dist/tfjs.esm.js"
}
]
},
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytes": 6986, "bytes": 6986,
"imports": [ "imports": [
@ -342,7 +350,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 9243, "bytes": 9241,
"imports": [] "imports": []
}, },
"src/sample.js": { "src/sample.js": {
@ -354,7 +362,7 @@
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 16894, "bytes": 17068,
"imports": [ "imports": [
{ {
"path": "src/log.js" "path": "src/log.js"
@ -362,6 +370,9 @@
{ {
"path": "dist/tfjs.esm.js" "path": "dist/tfjs.esm.js"
}, },
{
"path": "src/tfjs/backend.js"
},
{ {
"path": "src/face/facemesh.js" "path": "src/face/facemesh.js"
}, },
@ -408,14 +419,14 @@
"dist/human.js.map": { "dist/human.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 2640856 "bytes": 2123892
}, },
"dist/human.js": { "dist/human.js": {
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": { "inputs": {
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytesInOutput": 5096 "bytesInOutput": 5106
}, },
"src/face/box.js": { "src/face/box.js": {
"bytesInOutput": 1578 "bytesInOutput": 1578
@ -427,25 +438,25 @@
"bytesInOutput": 30817 "bytesInOutput": 30817
}, },
"src/face/facepipeline.js": { "src/face/facepipeline.js": {
"bytesInOutput": 9314 "bytesInOutput": 9329
}, },
"src/face/facemesh.js": { "src/face/facemesh.js": {
"bytesInOutput": 2182 "bytesInOutput": 2202
}, },
"src/profile.js": { "src/profile.js": {
"bytesInOutput": 846 "bytesInOutput": 846
}, },
"src/age/age.js": { "src/age/age.js": {
"bytesInOutput": 1169 "bytesInOutput": 1181
}, },
"src/gender/gender.js": { "src/gender/gender.js": {
"bytesInOutput": 1915 "bytesInOutput": 1929
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1795 "bytesInOutput": 1807
}, },
"src/embedding/embedding.js": { "src/embedding/embedding.js": {
"bytesInOutput": 1310 "bytesInOutput": 1318
}, },
"src/body/modelBase.js": { "src/body/modelBase.js": {
"bytesInOutput": 615 "bytesInOutput": 615
@ -475,22 +486,22 @@
"bytesInOutput": 1913 "bytesInOutput": 1913
}, },
"src/body/modelPoseNet.js": { "src/body/modelPoseNet.js": {
"bytesInOutput": 1558 "bytesInOutput": 1569
}, },
"src/body/posenet.js": { "src/body/posenet.js": {
"bytesInOutput": 832 "bytesInOutput": 832
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 2731 "bytesInOutput": 2742
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 4447 "bytesInOutput": 4459
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127032 "bytesInOutput": 127032
}, },
"src/hand/handpose.js": { "src/hand/handpose.js": {
"bytesInOutput": 2007 "bytesInOutput": 2022
}, },
"src/gesture/gesture.js": { "src/gesture/gesture.js": {
"bytesInOutput": 2463 "bytesInOutput": 2463
@ -499,16 +510,19 @@
"bytesInOutput": 13576 "bytesInOutput": 13576
}, },
"src/image.js": { "src/image.js": {
"bytesInOutput": 4008 "bytesInOutput": 4040
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 11134 "bytesInOutput": 11216
}, },
"src/log.js": { "src/log.js": {
"bytesInOutput": 266 "bytesInOutput": 266
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1487547 "bytesInOutput": 1525301
},
"src/tfjs/backend.js": {
"bytesInOutput": 938
}, },
"src/hand/box.js": { "src/hand/box.js": {
"bytesInOutput": 1473 "bytesInOutput": 1473
@ -526,7 +540,7 @@
"bytesInOutput": 21 "bytesInOutput": 21
} }
}, },
"bytes": 1797597 "bytes": 1836543
} }
} }
} }

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

6
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

52
dist/human.node.json vendored
View File

@ -8,6 +8,14 @@
"bytes": 1048, "bytes": 1048,
"imports": [] "imports": []
}, },
"src/tfjs/backend.js": {
"bytes": 1340,
"imports": [
{
"path": "dist/tfjs.esm.js"
}
]
},
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytes": 6986, "bytes": 6986,
"imports": [ "imports": [
@ -342,7 +350,7 @@
] ]
}, },
"config.js": { "config.js": {
"bytes": 9243, "bytes": 9241,
"imports": [] "imports": []
}, },
"src/sample.js": { "src/sample.js": {
@ -350,11 +358,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2323, "bytes": 2324,
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 16894, "bytes": 17068,
"imports": [ "imports": [
{ {
"path": "src/log.js" "path": "src/log.js"
@ -362,6 +370,9 @@
{ {
"path": "dist/tfjs.esm.js" "path": "dist/tfjs.esm.js"
}, },
{
"path": "src/tfjs/backend.js"
},
{ {
"path": "src/face/facemesh.js" "path": "src/face/facemesh.js"
}, },
@ -408,7 +419,7 @@
"dist/human.node-gpu.js.map": { "dist/human.node-gpu.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 760521 "bytes": 780897
}, },
"dist/human.node-gpu.js": { "dist/human.node-gpu.js": {
"imports": [], "imports": [],
@ -418,7 +429,7 @@
"bytesInOutput": 966 "bytesInOutput": 966
}, },
"src/face/blazeface.js": { "src/face/blazeface.js": {
"bytesInOutput": 5251 "bytesInOutput": 5261
}, },
"src/face/box.js": { "src/face/box.js": {
"bytesInOutput": 1628 "bytesInOutput": 1628
@ -430,25 +441,25 @@
"bytesInOutput": 30828 "bytesInOutput": 30828
}, },
"src/face/facepipeline.js": { "src/face/facepipeline.js": {
"bytesInOutput": 9364 "bytesInOutput": 9379
}, },
"src/face/facemesh.js": { "src/face/facemesh.js": {
"bytesInOutput": 2225 "bytesInOutput": 2245
}, },
"src/profile.js": { "src/profile.js": {
"bytesInOutput": 846 "bytesInOutput": 846
}, },
"src/age/age.js": { "src/age/age.js": {
"bytesInOutput": 1227 "bytesInOutput": 1239
}, },
"src/gender/gender.js": { "src/gender/gender.js": {
"bytesInOutput": 1997 "bytesInOutput": 2011
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1870 "bytesInOutput": 1882
}, },
"src/embedding/embedding.js": { "src/embedding/embedding.js": {
"bytesInOutput": 1361 "bytesInOutput": 1369
}, },
"src/body/modelBase.js": { "src/body/modelBase.js": {
"bytesInOutput": 652 "bytesInOutput": 652
@ -478,22 +489,22 @@
"bytesInOutput": 1918 "bytesInOutput": 1918
}, },
"src/body/modelPoseNet.js": { "src/body/modelPoseNet.js": {
"bytesInOutput": 1596 "bytesInOutput": 1607
}, },
"src/body/posenet.js": { "src/body/posenet.js": {
"bytesInOutput": 846 "bytesInOutput": 846
}, },
"src/hand/handdetector.js": { "src/hand/handdetector.js": {
"bytesInOutput": 2869 "bytesInOutput": 2880
}, },
"src/hand/handpipeline.js": { "src/hand/handpipeline.js": {
"bytesInOutput": 4491 "bytesInOutput": 4503
}, },
"src/hand/anchors.js": { "src/hand/anchors.js": {
"bytesInOutput": 127034 "bytesInOutput": 127034
}, },
"src/hand/handpose.js": { "src/hand/handpose.js": {
"bytesInOutput": 2049 "bytesInOutput": 2064
}, },
"src/gesture/gesture.js": { "src/gesture/gesture.js": {
"bytesInOutput": 2467 "bytesInOutput": 2467
@ -502,16 +513,19 @@
"bytesInOutput": 13558 "bytesInOutput": 13558
}, },
"src/image.js": { "src/image.js": {
"bytesInOutput": 4037 "bytesInOutput": 4069
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 11159 "bytesInOutput": 11243
}, },
"src/log.js": { "src/log.js": {
"bytesInOutput": 266 "bytesInOutput": 266
}, },
"src/tfjs/backend.js": {
"bytesInOutput": 1001
},
"src/hand/box.js": { "src/hand/box.js": {
"bytesInOutput": 1510 "bytesInOutput": 1512
}, },
"src/hand/util.js": { "src/hand/util.js": {
"bytesInOutput": 1790 "bytesInOutput": 1790
@ -526,7 +540,7 @@
"bytesInOutput": 20 "bytesInOutput": 20
} }
}, },
"bytes": 305516 "bytes": 306776
} }
} }
} }

418
dist/tfjs.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

10087
dist/tfjs.esm.json vendored

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,6 @@
import { log } from './log.js'; import { log } from './log.js';
import * as tf from '../dist/tfjs.esm.js'; import * as tf from '../dist/tfjs.esm.js';
import * as backend from './tfjs/backend.js';
import * as facemesh from './face/facemesh.js'; import * as facemesh from './face/facemesh.js';
import * as age from './age/age.js'; import * as age from './age/age.js';
import * as gender from './gender/gender.js'; import * as gender from './gender/gender.js';
@ -176,6 +177,11 @@ class Human {
if (!simd) log('warning: wasm simd support is not enabled'); if (!simd) log('warning: wasm simd support is not enabled');
} }
if (this.config.backend === 'humangl') {
log('registering humangl backend');
backend.register();
}
await tf.setBackend(this.config.backend); await tf.setBackend(this.config.backend);
tf.enableProdMode(); tf.enableProdMode();
/* debug mode is really too mcuh /* debug mode is really too mcuh

38
src/tfjs/backend.js Normal file
View File

@ -0,0 +1,38 @@
import * as tf from '../../dist/tfjs.esm.js';
export const config = {
name: 'humangl',
priority: 99,
canvas: null,
width: 1024,
height: 1024,
webGLattr: { // https://www.khronos.org/registry/webgl/specs/latest/1.0/#5.2
alpha: false,
antialias: false,
premultipliedAlpha: false,
preserveDrawingBuffer: false,
depth: false,
stencil: false,
failIfMajorPerformanceCaveat: false,
desynchronized: true,
},
};
export function register() {
if (!tf.findBackend(config.name)) {
config.canvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(config.width, config.height) : document.createElement('canvas');
const gl = config.canvas.getContext('webgl2', config.webGLattr);
tf.setWebGLContext(2, gl);
const ctx = new tf.GPGPUContext(gl);
tf.registerBackend(config.name, () => new tf.MathBackendWebGL(ctx), config.priority);
const kernels = tf.getKernelsForBackend('webgl');
kernels.forEach((kernelConfig) => {
const newKernelConfig = { ...kernelConfig, backendName: config.name };
tf.registerKernel(newKernelConfig);
});
tf.ENV.set('WEBGL_VERSION', 2);
tf.ENV.set('WEBGL_MAX_TEXTURE_SIZE', gl.getParameter(gl.MAX_TEXTURE_SIZE));
tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
}
}

View File

@ -3,13 +3,12 @@
// simplified // simplified
// { modules: 1061, moduleBytes: 3772720, outputBytes: 1531035 } // { modules: 1061, moduleBytes: 3772720, outputBytes: 1531035 }
export * from '@tensorflow/tfjs/dist/index.js'; // export * from '@tensorflow/tfjs/dist/index.js';
export * from '@tensorflow/tfjs-backend-wasm'; // export * from '@tensorflow/tfjs-backend-wasm';
// modular // modular
// { modules: 1064, moduleBytes: 3793219, outputBytes: 1535600 } // { modules: 1064, moduleBytes: 3793219, outputBytes: 1535600 }
/*
// get versions of all packages. // get versions of all packages.
import { version as tfjs } from '@tensorflow/tfjs/package.json'; import { version as tfjs } from '@tensorflow/tfjs/package.json';
import { version as versionCore } from '@tensorflow/tfjs-core/package.json'; import { version as versionCore } from '@tensorflow/tfjs-core/package.json';
@ -41,4 +40,3 @@ export const version = {
'tfjs-backend-webgl': version_webgl, 'tfjs-backend-webgl': version_webgl,
'tfjs-backend-wasm': version_wasm, 'tfjs-backend-wasm': version_wasm,
}; };
*/