diff --git a/assets/gl-bench.js b/assets/gl-bench.js
new file mode 100644
index 00000000..0fb6bc4f
--- /dev/null
+++ b/assets/gl-bench.js
@@ -0,0 +1,273 @@
+// downloaded from https://github.com/munrocket/gl-bench
+// this file is https://github.com/munrocket/gl-bench/blob/master/dist/gl-bench.module.js
+
+/*
+var UISVG = "
\n
\n
\n
\n
";
+
+var UICSS = "#gl-bench {\n position:absolute;\n left:0;\n top:0;\n z-index:1000;\n -webkit-user-select: none;\n -moz-user-select: none;\n user-select: none;\n}\n\n#gl-bench div {\n position: relative;\n display: block;\n margin: 4px;\n padding: 0 7px 0 10px;\n background: #6c6;\n border-radius: 15px;\n cursor: pointer;\n opacity: 0.9;\n}\n\n#gl-bench svg {\n height: 60px;\n margin: 0 -1px;\n}\n\n#gl-bench text {\n font-size: 12px;\n font-family: Helvetica,Arial,sans-serif;\n font-weight: 700;\n dominant-baseline: middle;\n text-anchor: middle;\n}\n\n#gl-bench .gl-mem {\n font-size: 9px;\n}\n\n#gl-bench line {\n stroke-width: 5;\n stroke: #112211;\n stroke-linecap: round;\n}\n\n#gl-bench polyline {\n fill: none;\n stroke: #112211;\n stroke-linecap: round;\n stroke-linejoin: round;\n stroke-width: 3.5;\n}\n\n#gl-bench rect {\n fill: #448844;\n}\n\n#gl-bench .opacity {\n stroke: #448844;\n}\n";
+*/
+
+const UISVG = `
+
+
+
+
+
+ `;
+
+const UICSS = `
+ #gl-bench { position: absolute; right:0; bottom:0; z-index:1000; -webkit-user-select: none; -moz-user-select: none; user-select: none; }
+ #gl-bench div { position: relative; display: block; margin: 4px; padding: 0 7px 0 10px; background: darkslategray; border-radius: 0.2rem; cursor: pointer; opacity: 0.9; }
+ #gl-bench svg { height: 60px; margin: 0 -1px; }
+ #gl-bench text { font-size: 12px; font-family: Helvetica,Arial,sans-serif; font-weight: 700; dominant-baseline: middle; text-anchor: middle; }
+ #gl-bench .gl-mem { font-size: 9px; fill: white; }
+ #gl-bench .gl-fps { font-size: 10px; fill: white; }
+ #gl-bench line { stroke-width: 5; stroke: white; stroke-linecap: round; }
+ #gl-bench polyline { fill: none; stroke: white; stroke-linecap: round; stroke-linejoin: round; stroke-width: 3.5; }
+ #gl-bench rect { fill: black; }
+ #gl-bench .opacity { stroke: black; }
+ `;
+
+class GLBench {
+
+ /** GLBench constructor
+ * @param { WebGLRenderingContext | WebGL2RenderingContext } gl context
+ * @param { Object | undefined } settings additional settings
+ */
+ constructor(gl, settings = {}) {
+ this.css = UICSS;
+ this.svg = UISVG;
+ this.paramLogger = () => {};
+ this.chartLogger = () => {};
+ this.chartLen = 20;
+ this.chartHz = 20;
+
+ this.names = [];
+ this.cpuAccums = [];
+ this.gpuAccums = [];
+ this.activeAccums = [];
+ this.chart = new Array(this.chartLen);
+ this.now = () => (performance && performance.now) ? performance.now() : Date.now();
+ this.updateUI = () => {
+ [].forEach.call(this.nodes['gl-gpu-svg'], node => {
+ node.style.display = this.trackGPU ? 'inline' : 'none';
+ });
+ };
+
+ Object.assign(this, settings);
+ this.detected = 0;
+ this.finished = [];
+ this.isFramebuffer = 0;
+ this.frameId = 0;
+
+ // 120hz device detection
+ let rafId, n = 0, t0;
+ let loop = (t) => {
+ if (++n < 20) {
+ rafId = requestAnimationFrame(loop);
+ } else {
+ this.detected = Math.ceil(1e3 * n / (t - t0) / 70);
+ cancelAnimationFrame(rafId);
+ }
+ if (!t0) t0 = t;
+ };
+ requestAnimationFrame(loop);
+
+ // attach gpu profilers
+ if (gl) {
+ const glFinish = async (t, activeAccums) =>
+ Promise.resolve(setTimeout(() => {
+ gl.getError();
+ const dt = this.now() - t;
+ activeAccums.forEach((active, i) => {
+ if (active) this.gpuAccums[i] += dt;
+ });
+ }, 0));
+
+ const addProfiler = (fn, self, target) => function() {
+ const t = self.now();
+ fn.apply(target, arguments);
+ if (self.trackGPU) self.finished.push(glFinish(t, self.activeAccums.slice(0)));
+ };
+
+ ['drawArrays', 'drawElements', 'drawArraysInstanced',
+ 'drawBuffers', 'drawElementsInstanced', 'drawRangeElements']
+ .forEach(fn => { if (gl[fn]) gl[fn] = addProfiler(gl[fn], this, gl); });
+
+ gl.getExtension = ((fn, self) => function() {
+ let ext = fn.apply(gl, arguments);
+ if (ext) {
+ ['drawElementsInstancedANGLE', 'drawBuffersWEBGL']
+ .forEach(fn => { if (ext[fn]) ext[fn] = addProfiler(ext[fn], self, ext); });
+ }
+ return ext;
+ })(gl.getExtension, this);
+ }
+
+ // init ui and ui loggers
+ if (!this.withoutUI) {
+ if (!this.dom) this.dom = document.body;
+ let elm = document.createElement('div');
+ elm.id = 'gl-bench';
+ this.dom.appendChild(elm);
+ this.dom.insertAdjacentHTML('afterbegin', '');
+ this.dom = elm;
+ this.dom.addEventListener('click', () => {
+ this.trackGPU = !this.trackGPU;
+ this.updateUI();
+ });
+
+ this.paramLogger = ((logger, dom, names) => {
+ const classes = ['gl-cpu', 'gl-gpu', 'gl-mem', 'gl-fps', 'gl-gpu-svg', 'gl-chart'];
+ const nodes = Object.assign({}, classes);
+ classes.forEach(c => nodes[c] = dom.getElementsByClassName(c));
+ this.nodes = nodes;
+ return (i, cpu, gpu, mem, fps, totalTime, frameId) => {
+ nodes['gl-cpu'][i].style.strokeDasharray = (cpu * 0.27).toFixed(0) + ' 100';
+ nodes['gl-gpu'][i].style.strokeDasharray = (gpu * 0.27).toFixed(0) + ' 100';
+ nodes['gl-mem'][i].innerHTML = names[i] ? names[i] : (mem ? 'mem: ' + mem.toFixed(0) + 'mb' : '');
+ nodes['gl-fps'][i].innerHTML = fps.toFixed(0) + ' FPS';
+ logger(names[i], cpu, gpu, mem, fps, totalTime, frameId);
+ }
+ })(this.paramLogger, this.dom, this.names);
+
+ this.chartLogger = ((logger, dom) => {
+ let nodes = { 'gl-chart': dom.getElementsByClassName('gl-chart') };
+ return (i, chart, circularId) => {
+ let points = '';
+ let len = chart.length;
+ for (let i = 0; i < len; i++) {
+ let id = (circularId + i + 1) % len;
+ if (chart[id] != undefined) {
+ points = points + ' ' + (55 * i / (len - 1)).toFixed(1) + ','
+ + (45 - chart[id] * 22 / 60 / this.detected).toFixed(1);
+ }
+ }
+ nodes['gl-chart'][i].setAttribute('points', points);
+ logger(this.names[i], chart, circularId);
+ }
+ })(this.chartLogger, this.dom);
+ }
+ }
+
+ /**
+ * Explicit UI add
+ * @param { string | undefined } name
+ */
+ addUI(name) {
+ if (this.names.indexOf(name) == -1) {
+ this.names.push(name);
+ if (this.dom) {
+ this.dom.insertAdjacentHTML('beforeend', this.svg);
+ this.updateUI();
+ }
+ this.cpuAccums.push(0);
+ this.gpuAccums.push(0);
+ this.activeAccums.push(false);
+ }
+ }
+
+ /**
+ * Increase frameID
+ * @param { number | undefined } now
+ */
+ nextFrame(now) {
+ this.frameId++;
+ const t = now ? now : this.now();
+
+ // params
+ if (this.frameId <= 1) {
+ this.paramFrame = this.frameId;
+ this.paramTime = t;
+ } else {
+ let duration = t - this.paramTime;
+ if (duration >= 1e3) {
+ const frameCount = this.frameId - this.paramFrame;
+ const fps = frameCount / duration * 1e3;
+ for (let i = 0; i < this.names.length; i++) {
+ const cpu = this.cpuAccums[i] / duration * 100,
+ gpu = this.gpuAccums[i] / duration * 100,
+ mem = (performance && performance.memory) ? performance.memory.usedJSHeapSize / (1 << 20) : 0;
+ this.paramLogger(i, cpu, gpu, mem, fps, duration, frameCount);
+ this.cpuAccums[i] = 0;
+ Promise.all(this.finished).then(() => {
+ this.gpuAccums[i] = 0;
+ this.finished = [];
+ });
+ }
+ this.paramFrame = this.frameId;
+ this.paramTime = t;
+ }
+ }
+
+ // chart
+ if (!this.detected || !this.chartFrame) {
+ this.chartFrame = this.frameId;
+ this.chartTime = t;
+ this.circularId = 0;
+ } else {
+ let timespan = t - this.chartTime;
+ let hz = this.chartHz * timespan / 1e3;
+ while (--hz > 0 && this.detected) {
+ const frameCount = this.frameId - this.chartFrame;
+ const fps = frameCount / timespan * 1e3;
+ this.chart[this.circularId % this.chartLen] = fps;
+ for (let i = 0; i < this.names.length; i++) {
+ this.chartLogger(i, this.chart, this.circularId);
+ }
+ this.circularId++;
+ this.chartFrame = this.frameId;
+ this.chartTime = t;
+ }
+ }
+ }
+
+ /**
+ * Begin named measurement
+ * @param { string | undefined } name
+ */
+ begin(name) {
+ this.updateAccums(name);
+ }
+
+ /**
+ * End named measure
+ * @param { string | undefined } name
+ */
+ end(name) {
+ this.updateAccums(name);
+ }
+
+ updateAccums(name) {
+ let nameId = this.names.indexOf(name);
+ if (nameId == -1) {
+ nameId = this.names.length;
+ this.addUI(name);
+ }
+
+ const t = this.now();
+ const dt = t - this.t0;
+ for (let i = 0; i < nameId + 1; i++) {
+ if (this.activeAccums[i]) {
+ this.cpuAccums[i] += dt;
+ }
+ } this.activeAccums[nameId] = !this.activeAccums[nameId];
+ this.t0 = t;
+ }
+
+}
+
+export default GLBench;
diff --git a/demo/browser.js b/demo/browser.js
index cd1ef008..2c452421 100644
--- a/demo/browser.js
+++ b/demo/browser.js
@@ -1,6 +1,7 @@
import Human from '../dist/human.esm.js';
import draw from './draw.js';
import Menu from './menu.js';
+import GLBench from '../assets/gl-bench.js';
const userConfig = {}; // add any user configuration overrides
@@ -39,12 +40,14 @@ const ui = {
drawThread: null,
framesDraw: 0,
framesDetect: 0,
+ bench: false,
};
// global variables
let menu;
let menuFX;
let worker;
+let bench;
let lastDetectedResult = {};
// helper function: translates json to human readable string
@@ -199,32 +202,29 @@ async function setupCamera() {
}
// wrapper for worker.postmessage that creates worker if one does not exist
-function webWorker(input, image, canvas) {
+function webWorker(input, image, canvas, timestamp) {
if (!worker) {
// create new webworker and add event handler only once
log('creating worker thread');
worker = new Worker(ui.worker, { type: 'module' });
- worker.warned = false;
// after receiving message from webworker, parse&draw results and send new frame for processing
worker.addEventListener('message', (msg) => {
- if (!worker.warned) {
- log('warning: cannot transfer canvas from worked thread');
- log('warning: image will not show filter effects');
- worker.warned = true;
- }
+ if (ui.bench) bench.end();
+ if (ui.bench) bench.nextFrame(timestamp);
lastDetectedResult = msg.data.result;
ui.framesDetect++;
if (!ui.drawThread) drawResults(input);
// eslint-disable-next-line no-use-before-define
- requestAnimationFrame(() => runHumanDetect(input, canvas));
+ requestAnimationFrame((now) => runHumanDetect(input, canvas, now));
});
}
// pass image data as arraybuffer to worker by reference to avoid copy
+ if (ui.bench) bench.begin();
worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height }, [image.data.buffer]);
}
// main processing function when input is webcam, can use direct invocation or web worker
-function runHumanDetect(input, canvas) {
+function runHumanDetect(input, canvas, timestamp) {
// if live video
const live = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
if (!live && input.srcObject) {
@@ -248,15 +248,18 @@ function runHumanDetect(input, canvas) {
ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
// perform detection in worker
- webWorker(input, data, canvas, userConfig);
+ webWorker(input, data, canvas, userConfig, timestamp);
} else {
+ if (ui.bench) bench.begin();
human.detect(input, userConfig).then((result) => {
+ if (ui.bench) bench.end();
+ if (ui.bench) bench.nextFrame(timestamp);
if (result.error) log(result.error);
else {
lastDetectedResult = result;
if (!ui.drawThread) drawResults(input);
ui.framesDetect++;
- requestAnimationFrame(() => runHumanDetect(input, canvas));
+ requestAnimationFrame((now) => runHumanDetect(input, canvas, now));
}
});
}
@@ -412,9 +415,30 @@ function setupMenu() {
menuFX.addBool('polaroid', human.config.filter, 'polaroid');
}
+async function setupMonitor() {
+ let gl = human.tf.engine().backend.gpgpu;
+ if (!gl) gl = document.getElementById('bench-canvas').getContext('webgl2');
+ if (!bench) {
+ bench = new GLBench(gl, {
+ trackGPU: true,
+ chartHz: 20,
+ chartLen: 50,
+ });
+ }
+ /*
+ function update(now) {
+ bench.nextFrame(now);
+ requestAnimationFrame(update);
+ }
+ requestAnimationFrame(update);
+ */
+ // class MathBackendWebGL extends tf.KernelBackend property gpgpu is gl context
+}
+
async function main() {
log('Human: demo starting ...');
setupMenu();
+ setupMonitor();
document.getElementById('log').innerText = `Human: version ${human.version} TensorFlow/JS: version ${human.tf.version_core}`;
// human.tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
// this is not required, just pre-loads all models
diff --git a/demo/index.html b/demo/index.html
index 0bdb62bc..5cff8d82 100644
--- a/demo/index.html
+++ b/demo/index.html
@@ -33,6 +33,7 @@
.samples-container { display: flex; flex-wrap: wrap; }
.video { display: none; }
.canvas { margin: 0 auto; }
+ .bench { position: absolute; right: 0; bottom: 0; }
.loader { width: 300px; height: 300px; border: 3px solid transparent; border-radius: 50%; border-top: 4px solid #f15e41; animation: spin 4s linear infinite; position: absolute; top: 30%; left: 50%; margin-left: -150px; z-index: 15; }
.loader::before, .loader::after { content: ""; position: absolute; top: 6px; bottom: 6px; left: 6px; right: 6px; border-radius: 50%; border: 4px solid transparent; }
.loader::before { border-top-color: #bad375; animation: 3s spin linear infinite; }
@@ -70,6 +71,7 @@
+