From 26fbe21ab0011219be7ebac02e9ca0a15f6874fa Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Nov 2020 14:45:59 -0500 Subject: [PATCH] ui redesign --- .eslintrc.json | 27 ++-- demo/browser.js | 252 ++++++++++++++++++----------------- {assets => demo}/gl-bench.js | 38 ++++-- demo/index.html | 39 ++++-- demo/menu.js | 26 ++-- src/human.js | 2 +- 6 files changed, 219 insertions(+), 165 deletions(-) rename {assets => demo}/gl-bench.js (90%) diff --git a/.eslintrc.json b/.eslintrc.json index f5e43f60..4be59b55 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -21,12 +21,17 @@ ], "ignorePatterns": [ "dist", "assets", "media", "models", "node_modules" ], "rules": { - "max-len": [1, 275, 3], "camelcase": "off", - "guard-for-in": "off", - "prefer-template":"off", - "import/extensions": "off", + "dot-notation": "off", "func-names": "off", + "guard-for-in": "off", + "import/extensions": "off", + "import/no-absolute-path": "off", + "import/no-extraneous-dependencies": "off", + "import/prefer-default-export": "off", + "max-len": [1, 275, 3], + "newline-per-chained-call": "off", + "no-async-promise-executor": "off", "no-await-in-loop": "off", "no-bitwise": "off", "no-case-declarations":"off", @@ -35,25 +40,21 @@ "no-mixed-operators": "off", "no-param-reassign":"off", "no-plusplus": "off", - "dot-notation": "off", + "no-regex-spaces": "off", "no-restricted-globals": "off", "no-restricted-syntax": "off", - "no-underscore-dangle": "off", "no-return-assign": "off", - "newline-per-chained-call": "off", + "no-underscore-dangle": "off", + "node/no-unpublished-import": "off", + "node/no-unpublished-require": "off", "node/no-unsupported-features/es-syntax": "off", "node/shebang": "off", "object-curly-newline": "off", "prefer-destructuring": "off", + "prefer-template":"off", "promise/always-return": "off", "promise/catch-or-return": "off", "promise/no-nesting": "off", - "no-async-promise-executor": "off", - "import/no-absolute-path": "off", - "import/no-extraneous-dependencies": "off", - "node/no-unpublished-import": "off", - "node/no-unpublished-require": "off", - "no-regex-spaces": "off", "radix": "off" } } \ No newline at end of file diff --git a/demo/browser.js b/demo/browser.js index 65126e0f..80061fe7 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -1,7 +1,7 @@ import Human from '../dist/human.esm.js'; import draw from './draw.js'; import Menu from './menu.js'; -import GLBench from '../assets/gl-bench.js'; +import GLBench from './gl-bench.js'; const userConfig = {}; // add any user configuration overrides @@ -11,10 +11,9 @@ const human = new Human(userConfig); const ui = { baseColor: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel baseBackground: 'rgba(50, 50, 50, 1)', // 'grey' - baseLabel: 'rgba(173, 216, 230, 0.9)', // 'lightblue' with dark alpha channel + baseLabel: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel baseFontProto: 'small-caps {size} "Segoe UI"', baseLineWidth: 12, - baseLineHeightProto: 2, crop: true, columns: 2, busy: false, @@ -38,7 +37,6 @@ const ui = { detectFPS: [], drawFPS: [], buffered: false, - bufferedFPSTarget: 0, drawThread: null, detectThread: null, framesDraw: 0, @@ -47,11 +45,9 @@ const ui = { }; // global variables -let menu; -let menuFX; +const menu = {}; let worker; let bench; -let sample; let lastDetectedResult = {}; // helper function: translates json to human readable string @@ -78,14 +74,16 @@ const status = (msg) => { document.getElementById('status').innerText = msg; }; -async function calcSimmilariry(faces) { - if (!faces || !faces[0] || (faces[0].embedding?.length !== 192)) return; - const current = faces[0].embedding; - const original = (sample && sample.face && sample.face[0] && sample.face[0].embedding) ? sample.face[0].embedding : null; - if (original && original.length === 192) { - const simmilarity = human.simmilarity(current, original); - document.getElementById('simmilarity').innerText = `simmilarity: ${Math.trunc(1000 * simmilarity) / 10}%`; +let original; +async function calcSimmilariry(result) { + document.getElementById('compare-container').style.display = human.config.face.embedding.enabled ? 'block' : 'none'; + if ((result?.face?.length > 0) && (result?.face[0].embedding?.length !== 192)) return; + if (!original) { + original = result; + document.getElementById('compare-canvas').getContext('2d').drawImage(original.canvas, 0, 0, 200, 200); } + const simmilarity = human.simmilarity(original.face[0].embedding, result.face[0].embedding); + document.getElementById('simmilarity').innerText = `simmilarity: ${Math.trunc(1000 * simmilarity) / 10}%`; } // draws processed results and starts processing of a next frame @@ -103,7 +101,7 @@ async function drawResults(input) { // console.log(result.performance); // draw fps chart - await menu.updateChart('FPS', ui.detectFPS); + await menu.process.updateChart('FPS', ui.detectFPS); // get updated canvas if (ui.buffered || !result.canvas) result.canvas = await human.image(input, userConfig); @@ -125,7 +123,7 @@ async function drawResults(input) { await draw.body(result.body, canvas, ui); await draw.hand(result.hand, canvas, ui); await draw.gesture(result.gesture, canvas, ui); - await calcSimmilariry(result.face); + await calcSimmilariry(result); // update log const engine = human.tf.engine(); @@ -145,14 +143,11 @@ async function drawResults(input) { ui.framesDraw++; ui.lastFrame = performance.now(); // if buffered, immediate loop but limit frame rate although it's going to run slower as JS is singlethreaded - if ((ui.bufferedFPSTarget === 0) && ui.buffered) { + if (ui.buffered) { ui.drawThread = requestAnimationFrame(() => drawResults(input, canvas)); - } else if ((ui.bufferedFPSTarget === 0) && ui.buffered && !ui.drawThread) { - log('starting buffered refresh'); - if (ui.bufferedFPSTarget > 0) ui.drawThread = setInterval(() => drawResults(input, canvas), 1000 / ui.bufferedFPSTarget); } else if (!ui.buffered && ui.drawThread) { log('stopping buffered refresh'); - clearTimeout(ui.drawThread); + cancelAnimationFrame(ui.drawThread); ui.drawThread = null; } } @@ -181,7 +176,7 @@ async function setupCamera() { video: { facingMode: ui.facing ? 'user' : 'environment', resizeMode: ui.crop ? 'crop-and-scale' : 'none' }, }; if (window.innerWidth > window.innerHeight) constraints.video.width = { ideal: window.innerWidth }; - else constraints.video.height = { ideal: window.innerHeight }; + else constraints.video.height = { ideal: (window.innerHeight - document.getElementById('menubar').offsetHeight) }; try { stream = await navigator.mediaDevices.getUserMedia(constraints); } catch (err) { @@ -209,8 +204,9 @@ async function setupCamera() { ui.menuWidth.input.setAttribute('value', video.width); ui.menuHeight.input.setAttribute('value', video.height); // silly font resizing for paint-on-canvas since viewport can be zoomed - const size = 14 + (6 * canvas.width / window.innerWidth); + const size = Math.trunc(window.devicePixelRatio * (8 + (4 * canvas.width / window.innerWidth))); ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`); + ui.baseLineHeight = size + 4; if (live) video.play(); // eslint-disable-next-line no-use-before-define if (live && !ui.detectThread) runHumanDetect(video, canvas); @@ -223,6 +219,20 @@ async function setupCamera() { }); } +function initPerfMonitor() { + if (!bench) { + const gl = null; + // cosnt gl = human.tf.engine().backend.gpgpu.gl; + // if (!gl) log('bench cannot get tensorflow webgl context'); + bench = new GLBench(gl, { + trackGPU: false, // this is really slow + chartHz: 20, + chartLen: 20, + }); + bench.begin(); + } +} + // wrapper for worker.postmessage that creates worker if one does not exist function webWorker(input, image, canvas, timestamp) { if (!worker) { @@ -233,8 +243,11 @@ function webWorker(input, image, canvas, timestamp) { worker.addEventListener('message', (msg) => { if (msg.data.result.performance && msg.data.result.performance.total) ui.detectFPS.push(1000 / msg.data.result.performance.total); if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift(); - if (ui.bench) bench.end(); - if (ui.bench) bench.nextFrame(timestamp); + if (ui.bench) { + if (!bench) initPerfMonitor(); + bench.nextFrame(timestamp); + } + if (document.getElementById('gl-bench')) document.getElementById('gl-bench').style.display = ui.bench ? 'block' : 'none'; lastDetectedResult = msg.data.result; ui.framesDetect++; if (!ui.drawThread) drawResults(input); @@ -243,7 +256,6 @@ function webWorker(input, image, canvas, timestamp) { }); } // pass image data as arraybuffer to worker by reference to avoid copy - if (ui.bench) bench.begin(); worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, userConfig }, [image.data.buffer]); } @@ -253,7 +265,7 @@ function runHumanDetect(input, canvas, timestamp) { const live = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused); if (!live && input.srcObject) { // stop ui refresh - if (ui.drawThread) clearTimeout(ui.drawThread); + if (ui.drawThread) cancelAnimationFrame(ui.drawThread); if (ui.detectThread) cancelAnimationFrame(ui.detectThread); ui.drawThread = null; ui.detectThread = null; @@ -272,19 +284,20 @@ function runHumanDetect(input, canvas, timestamp) { const offscreen = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(canvas.width, canvas.height) : document.createElement('canvas'); offscreen.width = canvas.width; offscreen.height = canvas.height; - const ctx = offscreen.getContext('2d'); ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); const data = ctx.getImageData(0, 0, canvas.width, canvas.height); // perform detection in worker webWorker(input, data, canvas, userConfig, timestamp); } else { - if (ui.bench) bench.begin(); human.detect(input, userConfig).then((result) => { if (result.performance && result.performance.total) ui.detectFPS.push(1000 / result.performance.total); if (ui.detectFPS.length > ui.maxFPSframes) ui.detectFPS.shift(); - if (ui.bench) bench.end(); - if (ui.bench) bench.nextFrame(timestamp); + if (ui.bench) { + if (!bench) initPerfMonitor(); + bench.nextFrame(timestamp); + } + if (document.getElementById('gl-bench')) document.getElementById('gl-bench').style.display = ui.bench ? 'block' : 'none'; if (result.error) log(result.error); else { lastDetectedResult = result; @@ -331,15 +344,19 @@ async function detectVideo() { document.getElementById('canvas').style.display = 'block'; const video = document.getElementById('video'); const canvas = document.getElementById('canvas'); - ui.baseLineHeight = ui.baseLineHeightProto; if ((video.srcObject !== null) && !video.paused) { document.getElementById('play').style.display = 'block'; + document.getElementById('btnStart').className = 'button button-start'; + document.getElementById('btnStart').innerHTML = 'start
video'; status('paused'); video.pause(); } else { await setupCamera(); document.getElementById('play').style.display = 'none'; + for (const m of Object.values(menu)) m.hide(); status(''); + document.getElementById('btnStart').className = 'button button-stop'; + document.getElementById('btnStart').innerHTML = 'pause
video'; video.play(); } if (!ui.detectThread) runHumanDetect(video, canvas); @@ -349,9 +366,9 @@ async function detectVideo() { async function detectSampleImages() { document.getElementById('play').style.display = 'none'; userConfig.videoOptimized = false; - const size = 12 + Math.trunc(12 * ui.columns * window.innerWidth / document.body.clientWidth); + const size = Math.trunc(window.devicePixelRatio * (8 + (4 * ui.columns))); ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`); - ui.baseLineHeight = ui.baseLineHeightProto * ui.columns; + ui.baseLineHeight = size + 2; document.getElementById('canvas').style.display = 'none'; document.getElementById('samples-container').style.display = 'block'; log('Running detection of sample images'); @@ -362,121 +379,116 @@ async function detectSampleImages() { } function setupMenu() { - document.getElementById('compare-container').style.display = human.config.face.embedding.enabled ? 'block' : 'none'; - menu = new Menu(document.body, '', { top: '1rem', right: '1rem' }); - const btn = menu.addButton('start video', 'pause video', () => detectVideo()); - menu.addButton('process images', 'process images', () => detectSampleImages()); - document.getElementById('play').addEventListener('click', () => btn.click()); + let x = []; + if (window.innerWidth > 800) { + // initial position of menu items, later it's calculated based on mouse coordinates + x = [`${document.getElementById('btnDisplay').offsetLeft - 50}px`, `${document.getElementById('btnImage').offsetLeft - 50}px`, `${document.getElementById('btnProcess').offsetLeft - 50}px`, `${document.getElementById('btnModel').offsetLeft - 50}px`]; + } else { + // absolute minimum spacing for menus + x = ['0rem', '11rem', '21.1rem', '33rem']; + } - menu.addHTML('
'); - menu.addList('backend', ['cpu', 'webgl', 'wasm'], human.config.backend, (val) => human.config.backend = val); - menu.addBool('async operations', human.config, 'async', (val) => human.config.async = val); - // menu.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val); - // menu.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val); - menu.addBool('use web worker', ui, 'useWorker'); - menu.addHTML('
'); - menu.addLabel('enabled models'); - menu.addBool('face detect', human.config.face, 'enabled'); - menu.addBool('face mesh', human.config.face.mesh, 'enabled'); - menu.addBool('face iris', human.config.face.iris, 'enabled'); - menu.addBool('face age', human.config.face.age, 'enabled'); - menu.addBool('face gender', human.config.face.gender, 'enabled'); - menu.addBool('face emotion', human.config.face.emotion, 'enabled'); - // menu.addBool('face compare', human.config.face.embedding, 'enabled', (val) => { - // human.config.face.embedding.enabled = val; - // document.getElementById('compare-container').style.display = human.config.face.embedding.enabled ? 'block' : 'none'; - // }); - menu.addBool('body pose', human.config.body, 'enabled'); - menu.addBool('hand pose', human.config.hand, 'enabled'); - menu.addBool('gesture analysis', human.config.gesture, 'enabled'); + menu.display = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[0] }); + menu.display.addBool('perf monitor', ui, 'bench', (val) => ui.bench = val); + menu.display.addBool('buffered output', ui, 'buffered', (val) => ui.buffered = val); + menu.display.addBool('crop & scale', ui, 'crop', () => setupCamera()); + menu.display.addBool('camera facing', ui, 'facing', () => setupCamera()); + menu.display.addHTML('
'); + menu.display.addBool('use 3D depth', ui, 'useDepth'); + menu.display.addBool('draw boxes', ui, 'drawBoxes'); + menu.display.addBool('draw polygons', ui, 'drawPolygons'); + menu.display.addBool('Fill Polygons', ui, 'fillPolygons'); + menu.display.addBool('draw points', ui, 'drawPoints'); - menu.addHTML('
'); - menu.addLabel('model parameters'); - menu.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => { + menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] }); + menu.image.addBool('enabled', human.config.filter, 'enabled'); + ui.menuWidth = menu.image.addRange('image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val)); + ui.menuHeight = menu.image.addRange('image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val)); + menu.image.addHTML('
'); + menu.image.addRange('brightness', human.config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => human.config.filter.brightness = parseFloat(val)); + menu.image.addRange('contrast', human.config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => human.config.filter.contrast = parseFloat(val)); + menu.image.addRange('sharpness', human.config.filter, 'sharpness', 0, 1.0, 0.05, (val) => human.config.filter.sharpness = parseFloat(val)); + menu.image.addRange('blur', human.config.filter, 'blur', 0, 20, 1, (val) => human.config.filter.blur = parseInt(val)); + menu.image.addRange('saturation', human.config.filter, 'saturation', -1.0, 1.0, 0.05, (val) => human.config.filter.saturation = parseFloat(val)); + menu.image.addRange('hue', human.config.filter, 'hue', 0, 360, 5, (val) => human.config.filter.hue = parseInt(val)); + menu.image.addRange('pixelate', human.config.filter, 'pixelate', 0, 32, 1, (val) => human.config.filter.pixelate = parseInt(val)); + menu.image.addHTML('
'); + menu.image.addBool('negative', human.config.filter, 'negative'); + menu.image.addBool('sepia', human.config.filter, 'sepia'); + menu.image.addBool('vintage', human.config.filter, 'vintage'); + menu.image.addBool('kodachrome', human.config.filter, 'kodachrome'); + menu.image.addBool('technicolor', human.config.filter, 'technicolor'); + menu.image.addBool('polaroid', human.config.filter, 'polaroid'); + + menu.process = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[2] }); + menu.process.addList('backend', ['cpu', 'webgl', 'wasm'], human.config.backend, (val) => human.config.backend = val); + menu.process.addBool('async operations', human.config, 'async', (val) => human.config.async = val); + menu.process.addBool('enable profiler', human.config, 'profile', (val) => human.config.profile = val); + menu.process.addBool('memory shield', human.config, 'deallocate', (val) => human.config.deallocate = val); + menu.process.addBool('use web worker', ui, 'useWorker'); + menu.process.addHTML('
'); + menu.process.addLabel('model parameters'); + menu.process.addRange('max objects', human.config.face.detector, 'maxFaces', 1, 50, 1, (val) => { human.config.face.detector.maxFaces = parseInt(val); human.config.body.maxDetections = parseInt(val); human.config.hand.maxHands = parseInt(val); }); - menu.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => { + menu.process.addRange('skip frames', human.config.face.detector, 'skipFrames', 0, 50, 1, (val) => { human.config.face.detector.skipFrames = parseInt(val); human.config.face.emotion.skipFrames = parseInt(val); human.config.face.age.skipFrames = parseInt(val); human.config.hand.skipFrames = parseInt(val); }); - menu.addRange('min confidence', human.config.face.detector, 'minConfidence', 0.0, 1.0, 0.05, (val) => { + menu.process.addRange('min confidence', human.config.face.detector, 'minConfidence', 0.0, 1.0, 0.05, (val) => { human.config.face.detector.minConfidence = parseFloat(val); human.config.face.gender.minConfidence = parseFloat(val); human.config.face.emotion.minConfidence = parseFloat(val); human.config.hand.minConfidence = parseFloat(val); }); - menu.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => { + menu.process.addRange('score threshold', human.config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => { human.config.face.detector.scoreThreshold = parseFloat(val); human.config.hand.scoreThreshold = parseFloat(val); human.config.body.scoreThreshold = parseFloat(val); }); - menu.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => { + menu.process.addRange('overlap', human.config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => { human.config.face.detector.iouThreshold = parseFloat(val); human.config.hand.iouThreshold = parseFloat(val); }); + menu.process.addHTML('
'); + menu.process.addButton('process sample images', 'process images', () => detectSampleImages()); + menu.process.addHTML('
'); + menu.process.addChart('FPS', 'FPS'); - menu.addHTML('
'); - menu.addChart('FPS', 'FPS'); + menu.models = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[3] }); + menu.models.addBool('face detect', human.config.face, 'enabled'); + menu.models.addBool('face mesh', human.config.face.mesh, 'enabled'); + menu.models.addBool('face iris', human.config.face.iris, 'enabled'); + menu.models.addBool('face age', human.config.face.age, 'enabled'); + menu.models.addBool('face gender', human.config.face.gender, 'enabled'); + menu.models.addBool('face emotion', human.config.face.emotion, 'enabled'); + menu.models.addHTML('
'); + menu.models.addBool('body pose', human.config.body, 'enabled'); + menu.models.addBool('hand pose', human.config.hand, 'enabled'); + menu.models.addHTML('
'); + menu.models.addBool('gestures', human.config.gesture, 'enabled'); + menu.models.addHTML('
'); + menu.models.addBool('face compare', human.config.face.embedding, 'enabled', (val) => { + original = null; + human.config.face.embedding.enabled = val; + }); - menuFX = new Menu(document.body, '', { top: '1rem', right: '18rem' }); - menuFX.addLabel('ui options'); - menuFX.addBool('buffered output', ui, 'buffered', (val) => ui.buffered = val); - menuFX.addBool('crop & scale', ui, 'crop', () => setupCamera()); - menuFX.addBool('camera front/back', ui, 'facing', () => setupCamera()); - menuFX.addBool('use 3D depth', ui, 'useDepth'); - menuFX.addBool('draw boxes', ui, 'drawBoxes'); - menuFX.addBool('draw polygons', ui, 'drawPolygons'); - menuFX.addBool('Fill Polygons', ui, 'fillPolygons'); - menuFX.addBool('draw points', ui, 'drawPoints'); - menuFX.addHTML('
'); - menuFX.addLabel('image processing'); - menuFX.addBool('enabled', human.config.filter, 'enabled'); - ui.menuWidth = menuFX.addRange('image width', human.config.filter, 'width', 0, 3840, 10, (val) => human.config.filter.width = parseInt(val)); - ui.menuHeight = menuFX.addRange('image height', human.config.filter, 'height', 0, 2160, 10, (val) => human.config.filter.height = parseInt(val)); - menuFX.addRange('brightness', human.config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => human.config.filter.brightness = parseFloat(val)); - menuFX.addRange('contrast', human.config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => human.config.filter.contrast = parseFloat(val)); - menuFX.addRange('sharpness', human.config.filter, 'sharpness', 0, 1.0, 0.05, (val) => human.config.filter.sharpness = parseFloat(val)); - menuFX.addRange('blur', human.config.filter, 'blur', 0, 20, 1, (val) => human.config.filter.blur = parseInt(val)); - menuFX.addRange('saturation', human.config.filter, 'saturation', -1.0, 1.0, 0.05, (val) => human.config.filter.saturation = parseFloat(val)); - menuFX.addRange('hue', human.config.filter, 'hue', 0, 360, 5, (val) => human.config.filter.hue = parseInt(val)); - menuFX.addRange('pixelate', human.config.filter, 'pixelate', 0, 32, 1, (val) => human.config.filter.pixelate = parseInt(val)); - menuFX.addBool('negative', human.config.filter, 'negative'); - menuFX.addBool('sepia', human.config.filter, 'sepia'); - menuFX.addBool('vintage', human.config.filter, 'vintage'); - menuFX.addBool('kodachrome', human.config.filter, 'kodachrome'); - menuFX.addBool('technicolor', human.config.filter, 'technicolor'); - menuFX.addBool('polaroid', human.config.filter, 'polaroid'); -} - -async function setupMonitor() { - let gl = human.tf.engine().backend.gpgpu; - if (!gl) gl = document.getElementById('bench-canvas').getContext('webgl2'); - if (!bench) { - bench = new GLBench(gl, { - trackGPU: true, - chartHz: 20, - chartLen: 20, - }); - } - /* - function update(now) { - bench.nextFrame(now); - requestAnimationFrame(update); - } - requestAnimationFrame(update); - */ - // class MathBackendWebGL extends tf.KernelBackend property gpgpu is gl context + document.getElementById('btnDisplay').addEventListener('click', (evt) => menu.display.toggle(evt)); + document.getElementById('btnImage').addEventListener('click', (evt) => menu.image.toggle(evt)); + document.getElementById('btnProcess').addEventListener('click', (evt) => menu.process.toggle(evt)); + document.getElementById('btnModel').addEventListener('click', (evt) => menu.models.toggle(evt)); + document.getElementById('btnStart').addEventListener('click', () => detectVideo()); + document.getElementById('play').addEventListener('click', () => detectVideo()); } async function main() { log('demo starting ...'); setupMenu(); - setupMonitor(); - document.getElementById('log').innerText = `Human: version ${human.version} TensorFlow/JS: version ${human.tf.version_core}`; + document.getElementById('log').innerText = `Human: version ${human.version}`; // human.tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // this is not required, just pre-loads all models if (ui.modelsPreload && !ui.useWorker) { @@ -486,7 +498,7 @@ async function main() { // this is not required, just pre-warms all models for faster initial inference if (ui.modelsWarmup && !ui.useWorker) { status('initializing'); - sample = await human.warmup(userConfig, document.getElementById('sample-image')); + await human.warmup(userConfig); } status('human: ready'); document.getElementById('loader').style.display = 'none'; diff --git a/assets/gl-bench.js b/demo/gl-bench.js similarity index 90% rename from assets/gl-bench.js rename to demo/gl-bench.js index e7cb26bd..72ffffcb 100644 --- a/assets/gl-bench.js +++ b/demo/gl-bench.js @@ -1,9 +1,11 @@ -// modified based on: https://github.com/munrocket/gl-bench +/* eslint-disable max-len */ + +// based on: https://github.com/munrocket/gl-bench const UICSS = ` #gl-bench { position: absolute; right: 1rem; bottom: 1rem; z-index:1000; -webkit-user-select: none; -moz-user-select: none; user-select: none; } #gl-bench div { position: relative; display: block; margin: 4px; padding: 0 7px 0 10px; background: darkslategray; border-radius: 0.2rem; cursor: pointer; opacity: 0.9; } - #gl-bench svg { height: 60px; margin: 0 4px 0px 4px; } + #gl-bench svg { height: 60px; margin: 0 0px 0px 4px; } #gl-bench text { font-size: 16px; font-family: 'Lato', 'Segoe UI'; dominant-baseline: middle; text-anchor: middle; } #gl-bench .gl-mem { font-size: 12px; fill: white; } #gl-bench .gl-fps { font-size: 13px; fill: white; } @@ -17,7 +19,7 @@ const UISVG = `
00 FPS - + @@ -87,21 +89,41 @@ class GLBench { }); }, 0)); - const addProfiler = (fn, self, target) => function () { + const addProfiler = (fn, self, target) => { const t = self.now(); // eslint-disable-next-line prefer-rest-params fn.apply(target, arguments); if (self.trackGPU) self.finished.push(glFinish(t, self.activeAccums.slice(0))); }; - ['drawArrays', 'drawElements', 'drawArraysInstanced', 'drawBuffers', 'drawElementsInstanced', 'drawRangeElements'].forEach((fn) => { if (gl[fn]) gl[fn] = addProfiler(gl[fn], this, gl); }); + /* ['drawArrays', 'drawElements', 'drawArraysInstanced', 'drawBuffers', 'drawElementsInstanced', 'drawRangeElements'].forEach((fn) => { + if (gl[fn]) { + gl[fn] = addProfiler(gl[fn], this, gl); + } + }); + */ + const fn = 'drawElements'; + if (gl[fn]) { + gl[fn] = addProfiler(gl[fn], this, gl); + } else { + // eslint-disable-next-line no-console + console.log('bench: cannot attach to webgl function'); + } - gl.getExtension = ((fn, self) => function () { + /* + gl.getExtension = ((fn, self) => { // eslint-disable-next-line prefer-rest-params const ext = fn.apply(gl, arguments); - if (ext) ['drawElementsInstancedANGLE', 'drawBuffersWEBGL'].forEach((fn2) => { if (ext[fn2]) ext[fn2] = addProfiler(ext[fn2], self, ext); }); + if (ext) { + ['drawElementsInstancedANGLE', 'drawBuffersWEBGL'].forEach((fn2) => { + if (ext[fn2]) { + ext[fn2] = addProfiler(ext[fn2], self, ext); + } + }); + } return ext; })(gl.getExtension, this); + */ } // init ui and ui loggers @@ -127,7 +149,7 @@ class GLBench { nodes['gl-gpu'][i].style.strokeDasharray = (gpu * 0.27).toFixed(0) + ' 100'; // eslint-disable-next-line no-nested-ternary nodes['gl-mem'][i].innerHTML = names[i] ? names[i] : (mem ? 'mem: ' + mem.toFixed(0) + 'mb' : ''); - nodes['gl-fps'][i].innerHTML = fps.toFixed(0) + ' FPS'; + nodes['gl-fps'][i].innerHTML = 'FPS: ' + fps.toFixed(1); logger(names[i], cpu, gpu, mem, fps, totalTime, frameId); }; })(this.paramLogger, this.dom, this.names); diff --git a/demo/index.html b/demo/index.html index 1e12b8c6..9f182a60 100644 --- a/demo/index.html +++ b/demo/index.html @@ -19,22 +19,26 @@
- - + +
@@ -67,16 +80,22 @@
+
-
diff --git a/demo/menu.js b/demo/menu.js index e41845f8..6d44013e 100644 --- a/demo/menu.js +++ b/demo/menu.js @@ -19,15 +19,15 @@ function createCSS() { if (CSScreated) return; const css = ` :root { --rounded: 0.2rem; } - .menu { position: absolute; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; + .menu { position: absolute; top: 0rem; right: 0; width: fit-content; padding: 0 0.2rem 0 0.2rem; line-height: 1.8rem; z-index: 10; box-shadow: 0 0 8px dimgrey; background: ${theme.background}; border-radius: var(--rounded); border-color: black; border-style: solid; border-width: thin; } .menu:hover { box-shadow: 0 0 8px ${theme.hover}; } .menu-container { display: block; max-height: 100vh; } .menu-container-fadeout { max-height: 0; overflow: hidden; transition: max-height, 0.5s ease; } .menu-container-fadein { max-height: 100vh; overflow: hidden; transition: max-height, 0.5s ease; } - .menu-item { display: flex; white-space: nowrap; padding: 0.2rem; width: max-content; cursor: default; } - .menu-title { text-align: right; cursor: pointer; } + .menu-item { display: flex; white-space: nowrap; padding: 0.2rem; cursor: default; width: 100%; } + .menu-title { cursor: pointer; } .menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) } .menu-label { padding: 0; font-weight: 800; } @@ -39,12 +39,12 @@ function createCSS() { .menu-chart-title { padding: 0; font-size: 0.8rem; font-weight: 800; align-items: center} .menu-chart-canvas { background: transparent; margin: 0.2rem 0 0.2rem 0.6rem; } - .menu-button { border: 0; background: ${theme.buttonBackground}; width: 100%; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; + .menu-button { border: 0; background: ${theme.buttonBackground}; width: -webkit-fill-available; padding: 8px; margin: 8px; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; border-radius: var(--rounded); justify-content: center; font-family: inherit; font-variant: inherit; font-size: 1rem; font-weight: 800; } .menu-button:hover { background: ${theme.buttonHover}; box-shadow: 4px 4px 4px 0 black; } .menu-button:focus { outline: none; } - .menu-checkbox { width: 2.8rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: var(--rounded); } + .menu-checkbox { width: 2.8rem; height: 1rem; background: ${theme.itemBackground}; margin: 0.5rem 0.5rem 0 0; position: relative; border-radius: var(--rounded); } .menu-checkbox:after { content: 'OFF'; color: ${theme.checkboxOff}; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } .menu-checkbox:before { content: 'ON'; color: ${theme.checkboxOn}; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } .menu-checkbox-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: ${theme.checkboxOff}; @@ -53,14 +53,14 @@ function createCSS() { input[type=checkbox] { visibility: hidden; } input[type=checkbox]:checked + label { left: 1.4rem; background: ${theme.checkboxOn}; } - .menu-range { margin: 0 0.8rem 0 0; width: 5rem; background: transparent; color: ${theme.rangeBackground}; } + .menu-range { margin: 0.2rem 0.5rem 0 0; width: 3.5rem; background: transparent; color: ${theme.rangeBackground}; } .menu-range:before { color: ${theme.rangeLabel}; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; content: attr(value); } input[type=range] { -webkit-appearance: none; } input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; } input[type=range]::-moz-range-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; } - input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; } - input[type=range]::-moz-range-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; } + input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; } + input[type=range]::-moz-range-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; } .svg-background { fill:darkslategrey; cursor:pointer; opacity: 0.6; } .svg-foreground { fill:white; cursor:pointer; opacity: 0.8; } @@ -106,7 +106,7 @@ class Menu { `; - elTitle.innerHTML = `${title}${svg}`; + if (title) elTitle.innerHTML = `${title}${svg}`; this.menu.appendChild(elTitle); elTitle.addEventListener('click', () => { this.container.classList.toggle('menu-container-fadeout'); @@ -152,9 +152,9 @@ class Menu { this.container.classList.toggle('menu-container-fadein'); if (this.container.classList.contains('menu-container-fadein') && evt) { const x = evt.x || (evt.touches && evt.touches[0] ? evt.touches[0].pageX : null); - const y = evt.y || (evt.touches && evt.touches[0] ? evt.touches[0].pageY : null); - if (x) this.menu.style.left = `${x - 105}px`; - if (y) this.menu.style.top = '5.5rem'; // `${evt.y + 55}px`; + // const y = evt.y || (evt.touches && evt.touches[0] ? evt.touches[0].pageY : null); + if (x) this.menu.style.left = `${x - (this.menu.offsetWidth / 2)}px`; + // if (y) this.menu.style.top = '5.5rem'; // `${evt.y + 55}px`; if (this.menu.offsetLeft < 0) this.menu.style.left = 0; if ((this.menu.offsetLeft + this.menu.offsetWidth) > window.innerWidth) { this.menu.style.left = null; @@ -279,7 +279,7 @@ class Menu { else this.addValue(title, val); } - addChart(title, id, width = 200, height = 40, color) { + addChart(title, id, width = 150, height = 40, color) { if (color) theme.chartColor = color; const el = document.createElement('div'); el.className = 'menu-item menu-chart-title'; diff --git a/src/human.js b/src/human.js index f348a4fe..d26b54c7 100644 --- a/src/human.js +++ b/src/human.js @@ -116,8 +116,8 @@ class Human { if (userConfig) this.config = mergeDeep(this.config, userConfig); if (this.firstRun) { - this.checkBackend(true); this.log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`); + this.checkBackend(true); this.log('configuration:', this.config); this.log('flags:', tf.ENV.flags); this.firstRun = false;