diff --git a/demo/browser.js b/demo/browser.js index 81869c61..82c342bd 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -1,31 +1,32 @@ -/* global QuickSettings */ - import human from '../dist/human.esm.js'; import draw from './draw.js'; +import Menu from './menu.js'; // ui options const ui = { - baseColor: 'rgba(255, 200, 255, 0.3)', - baseLabel: 'rgba(255, 200, 255, 0.9)', + baseColor: 'rgba(173, 216, 230, 0.3)', // this is 'lightblue', just with alpha channel + baseLabel: 'rgba(173, 216, 230, 0.9)', baseFontProto: 'small-caps {size} "Segoe UI"', baseLineWidth: 16, baseLineHeightProto: 2, - columns: 3, + columns: 2, busy: false, - facing: 'user', + facing: true, useWorker: false, worker: 'worker.js', - samples: ['../assets/sample1.jpg', '../assets/sample2.jpg', '../assets/sample3.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample6.jpg'], + samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'], drawBoxes: true, drawPoints: false, drawPolygons: true, fillPolygons: true, useDepth: true, console: true, + maxFrames: 10, }; // configuration overrides const config = { + backend: 'webgl', // if you want to use 'wasm' backend, enable script load of tf and tf-backend-wasm in index.html face: { enabled: true, detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, @@ -40,7 +41,7 @@ const config = { }; // global variables -let settings; +let menu; let worker; let timeStamp; const fps = []; @@ -63,12 +64,11 @@ const log = (...msg) => { }; // draws processed results and starts processing of a next frame -async function drawResults(input, result, canvas) { +function drawResults(input, result, canvas) { // update fps - settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp))); fps.push(1000 / (performance.now() - timeStamp)); - if (fps.length > 20) fps.shift(); - settings.setValue('FPS', Math.round(10 * fps.reduce((a, b) => a + b) / fps.length) / 10); + if (fps.length > ui.maxFrames) fps.shift(); + menu.updateChart('FPS', fps); // eslint-disable-next-line no-use-before-define requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop @@ -81,7 +81,7 @@ async function drawResults(input, result, canvas) { draw.body(result.body, canvas, ui); draw.hand(result.hand, canvas, ui); // update log - const engine = await human.tf.engine(); + const engine = human.tf.engine(); const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`; const gpu = engine.backendInstance ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : ''; document.getElementById('log').innerText = ` @@ -98,7 +98,7 @@ async function setupCamera() { const canvas = document.getElementById('canvas'); const output = document.getElementById('log'); const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false; - let msg = `Setting up camera: live: ${live} facing: ${ui.facing}`; + let msg = `Setting up camera: live: ${live} facing: ${ui.facing ? 'front' : 'back'}`; output.innerText += `\n${msg}`; log(msg); // setup webcam. note that navigator.mediaDevices requires that page is accessed via https @@ -112,7 +112,7 @@ async function setupCamera() { try { stream = await navigator.mediaDevices.getUserMedia({ audio: false, - video: { facingMode: ui.facing, width: window.innerWidth, height: window.innerHeight }, + video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight }, }); } catch (err) { output.innerText += '\nCamera permission denied'; @@ -150,7 +150,7 @@ function webWorker(input, image, canvas) { } // main processing function when input is webcam, can use direct invocation or web worker -async function runHumanDetect(input, canvas) { +function runHumanDetect(input, canvas) { timeStamp = performance.now(); // perform detect if live video or not video at all if (input.srcObject) { @@ -170,36 +170,23 @@ async function runHumanDetect(input, canvas) { // perform detection in worker webWorker(input, data, canvas); } else { - let result = {}; - try { - // perform detection - result = await human.detect(input, config); - } catch (err) { - log('Error during execution:', err.message); - } - if (result.error) log(result.error); - else drawResults(input, result, canvas); + human.detect(input, config).then((result) => { + if (result.error) log(result.error); + else drawResults(input, result, canvas); + }); } } } // main processing function when input is image, can use direct invocation or web worker async function processImage(input) { - const cfg = { - backend: 'webgl', - console: true, - face: { - enabled: true, - detector: { maxFaces: 10, skipFrames: 0, minConfidence: 0.1, iouThreshold: 0.3, scoreThreshold: 0.3 }, - mesh: { enabled: true }, - iris: { enabled: true }, - age: { enabled: true, skipFrames: 0 }, - gender: { enabled: true }, - emotion: { enabled: true, minConfidence: 0.1, useGrayscale: true }, - }, - body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, - hand: { enabled: true, skipFrames: 0, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.5 }, - }; + // must be zero for images + config.face.detector.skipFrames = 0; + config.face.emotion.skipFrames = 0; + config.face.age.skipFrames = 0; + config.hand.skipFrames = 0; + + timeStamp = performance.now(); return new Promise((resolve) => { const image = document.getElementById('image'); image.onload = async () => { @@ -209,11 +196,13 @@ async function processImage(input) { image.height = image.naturalHeight; canvas.width = image.naturalWidth; canvas.height = image.naturalHeight; - const result = await human.detect(image, cfg); - await drawResults(image, result, canvas); + const result = await human.detect(image, config); + drawResults(image, result, canvas); const thumb = document.createElement('canvas'); - thumb.width = window.innerWidth / (ui.columns + 0.02); + thumb.width = (window.innerWidth - menu.width) / (ui.columns + 0.1); thumb.height = canvas.height / (window.innerWidth / thumb.width); + thumb.style.margin = '8px'; + thumb.style.boxShadow = '4px 4px 4px 0 dimgrey'; const ctx = thumb.getContext('2d'); ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, thumb.width, thumb.height); document.getElementById('samples').appendChild(thumb); @@ -253,74 +242,68 @@ async function detectSampleImages() { for (const sample of ui.samples) await processImage(sample); } -// setup settings panel -function setupUI() { - settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); - const style = document.createElement('style'); - style.innerHTML = ` - .qs_main { font: 1rem "Segoe UI"; } - .qs_label { font: 0.8rem "Segoe UI"; } - .qs_content { background: darkslategray; } - .qs_container { background: transparent; color: white; margin: 6px; padding: 6px; } - .qs_checkbox_label { top: 2px; } - .qs_button { width: -webkit-fill-available; font: 1rem "Segoe UI"; cursor: pointer; } - `; - document.getElementsByTagName('head')[0].appendChild(style); - settings.addButton('Play/Pause WebCam', () => detectVideo()); - settings.addButton('Process Images', () => detectSampleImages()); - settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => config.backend = val.value); - settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title'); - settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val); - settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val); - settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val); - settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val); - settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val); - settings.addBoolean('Face Emotion', config.face.emotion.enabled, (val) => config.face.emotion.enabled = val); - settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val); - settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val); - settings.addHTML('title', 'Model Parameters'); settings.hideTitle('title'); - settings.addRange('Max Objects', 1, 20, 5, 1, (val) => { +function setupMenu() { + menu = new Menu(document.body); + menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt)); + menu.addButton('Process Images', 'Process Images', () => detectSampleImages()); + + menu.addHTML('
'); + menu.addLabel('Enabled Models'); + menu.addBool('Face Detect', config.face, 'enabled'); + menu.addBool('Face Mesh', config.face.mesh, 'enabled'); + menu.addBool('Face Iris', config.face.iris, 'enabled'); + menu.addBool('Face Age', config.face.age, 'enabled'); + menu.addBool('Face Gender', config.face.gender, 'enabled'); + menu.addBool('Face Emotion', config.face.emotion, 'enabled'); + menu.addBool('Body Pose', config.body, 'enabled'); + menu.addBool('Hand Pose', config.hand, 'enabled'); + + menu.addHTML('
'); + menu.addLabel('Model Parameters'); + menu.addRange('Max Objects', config.face.detector, 'maxFaces', 0, 50, 1, (val) => { config.face.detector.maxFaces = parseInt(val); config.body.maxDetections = parseInt(val); + config.hand.maxHands = parseInt(val); }); - settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => { + menu.addRange('Skip Frames', config.face.detector, 'skipFrames', 0, 50, 1, (val) => { config.face.detector.skipFrames = parseInt(val); config.face.emotion.skipFrames = parseInt(val); config.face.age.skipFrames = parseInt(val); config.hand.skipFrames = parseInt(val); }); - settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => { + menu.addRange('Min Confidence', config.face.detector, 'minConfidence', 0.0, 1.0, 0.05, (val) => { config.face.detector.minConfidence = parseFloat(val); config.face.emotion.minConfidence = parseFloat(val); config.hand.minConfidence = parseFloat(val); }); - settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => { + menu.addRange('Score Threshold', config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => { config.face.detector.scoreThreshold = parseFloat(val); config.hand.scoreThreshold = parseFloat(val); config.body.scoreThreshold = parseFloat(val); }); - settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => { + menu.addRange('IOU Threshold', config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => { config.face.detector.iouThreshold = parseFloat(val); config.hand.iouThreshold = parseFloat(val); }); - settings.addHTML('title', 'UI Options'); settings.hideTitle('title'); - settings.addBoolean('Use Web Worker', ui.useWorker, (val) => ui.useWorker = val); - settings.addBoolean('Camera Front/Back', true, (val) => { - ui.facing = val ? 'user' : 'environment'; - setupCamera(); - }); - settings.addBoolean('Use 3D Depth', ui.useDepth, (val) => ui.useDepth = val); - settings.addBoolean('Draw Boxes', ui.drawBoxes, (val) => ui.drawBoxes = val); - settings.addBoolean('Draw Points', ui.drawPoints, (val) => ui.drawPoints = val); - settings.addBoolean('Draw Polygons', ui.drawPolygons, (val) => ui.drawPolygons = val); - settings.addBoolean('Fill Polygons', ui.fillPolygons, (val) => ui.fillPolygons = val); - settings.addHTML('line1', '
'); settings.hideTitle('line1'); - settings.addRange('FPS', 0, 100, 0, 1); + + menu.addHTML('
'); + menu.addLabel('UI Options'); + menu.addBool('Use Web Worker', ui, 'useWorker'); + menu.addBool('Camera Front/Back', ui, 'facing', () => setupCamera()); + menu.addBool('Use 3D Depth', ui, 'useDepth'); + menu.addBool('Draw Boxes', ui, 'drawBoxes'); + menu.addBool('Draw Points', ui, 'drawPoints'); + menu.addBool('Draw Polygons', ui, 'drawPolygons'); + menu.addBool('Fill Polygons', ui, 'fillPolygons'); + + menu.addHTML('
'); + menu.addValue('State', ''); + menu.addChart('FPS', 'FPS'); } async function main() { log('Human demo starting ...'); - setupUI(); + setupMenu(); const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`; document.getElementById('log').innerText += '\n' + msg; log(msg); diff --git a/demo/index.html b/demo/index.html index 3e6e2955..0d9270c8 100644 --- a/demo/index.html +++ b/demo/index.html @@ -14,7 +14,6 @@ - diff --git a/demo/menu.js b/demo/menu.js new file mode 100644 index 00000000..c6f47c76 --- /dev/null +++ b/demo/menu.js @@ -0,0 +1,166 @@ +const css = ` + .menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); } + .menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; } + .menu-title { padding: 0; } + .menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) } + .menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; } + + .menu-chart-title { align-items: center; } + .menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; } + + .menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; } + .menu-button:hover { background: lightgreen; } + + .menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; } + .menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } + .menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; } + input[type=checkbox] { visibility: hidden; } + input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; } + + .menu-range { margin: 0 0.8rem 0 0; width: 5rem; background: transparent; color: lightblue; } + .menu-range:before { content: attr(value); color: white; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; } + input[type=range] { -webkit-appearance: none; } + input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: black; border-radius: 1rem; border: 1px; } + input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: 1rem; background: lightblue; cursor: pointer; -webkit-appearance: none; } + `; + +function createCSS() { + const el = document.createElement('style'); + el.innerHTML = css; + document.getElementsByTagName('head')[0].appendChild(el); +} + +function createElem(parent) { + const el = document.createElement('div'); + el.id = 'menu'; + el.className = 'menu-container'; + if (typeof parent === 'object') parent.appendChild(el); + else document.getElementById(parent).appendChild(el); + return el; +} + +class Menu { + constructor(parent) { + createCSS(); + this.menu = createElem(parent); + this._id = 0; + this._maxFPS = 0; + } + + get newID() { + this._id++; + return `menu-${this._id}`; + } + + get ID() { + return `menu-${this._id}`; + } + + get width() { + return this.menu.offsetWidth; + } + + get height() { + return this.menu.offsetHeight; + } + + async addLabel(title) { + const el = document.createElement('div'); + el.className = 'menu menu-title'; + el.id = this.newID; + el.innerHTML = title; + this.menu.appendChild(el); + } + + async addBool(title, object, variable, callback) { + const el = document.createElement('div'); + el.className = 'menu'; + el.innerHTML = `
${title}`; + this.menu.appendChild(el); + document.getElementById(this.ID).addEventListener('change', (evt) => { + object[variable] = evt.target.checked; + if (callback) callback(evt.target.checked); + }); + } + + async addRange(title, object, variable, min, max, step, callback) { + const el = document.createElement('div'); + el.className = 'menu'; + el.innerHTML = `${title}`; + this.menu.appendChild(el); + document.getElementById(this.ID).addEventListener('change', (evt) => { + object[variable] = evt.target.value; + evt.target.setAttribute('value', evt.target.value); + if (callback) callback(evt.target.value); + }); + } + + async addHTML(html) { + const el = document.createElement('div'); + el.className = 'menu'; + el.id = this.newID; + if (html) el.innerHTML = html; + this.menu.appendChild(el); + } + + async addButton(titleOn, titleOff, callback) { + const el = document.createElement('button'); + el.className = 'menu menu-button'; + el.type = 'button'; + el.id = this.newID; + el.innerText = titleOn; + this.menu.appendChild(el); + document.getElementById(this.ID).addEventListener('click', () => { + if (el.innerText === titleOn) el.innerText = titleOff; + else el.innerText = titleOn; + if (callback) callback(el.innerText !== titleOn); + }); + } + + async addValue(title, val) { + const el = document.createElement('div'); + el.className = 'menu'; + el.id = title; + el.innerText = `${title}: ${val}`; + this.menu.appendChild(el); + } + + // eslint-disable-next-line class-methods-use-this + async updateValue(title, val) { + const el = document.getElementById(title); + el.innerText = `${title}: ${val}`; + } + + async addChart(title, id) { + const el = document.createElement('div'); + el.className = 'menu menu-chart-title'; + el.id = this.newID; + el.innerHTML = `${title}`; + this.menu.appendChild(el); + } + + // eslint-disable-next-line class-methods-use-this + async updateChart(id, values) { + if (!values || (values.length === 0)) return; + const canvas = document.getElementById(`menu-canvas-${id}`); + if (!canvas) return; + const ctx = canvas.getContext('2d'); + ctx.fillStyle = 'darkslategray'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + const width = canvas.width / values.length; + const max = 1 + Math.max(...values); + const height = canvas.height / max; + for (const i in values) { + const gradient = ctx.createLinearGradient(0, (max - values[i]) * height, 0, 0); + gradient.addColorStop(0.1, 'lightblue'); + gradient.addColorStop(0.4, 'darkslategray'); + ctx.fillStyle = gradient; + ctx.fillRect(i * width, 0, width - 4, canvas.height); + ctx.fillStyle = 'black'; + ctx.font = '12px "Segoe UI"'; + ctx.fillText(Math.round(values[i]), i * width, canvas.height - 2, width); + } + } +} + +export default Menu;