From 6de5ed066348a65dc1e14b0861366d42909567e6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 15 Oct 2020 18:16:05 -0400 Subject: [PATCH] reduced web worker latency --- demo/demo-esm-webworker.js | 5 ++++- demo/demo-esm.js | 46 ++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/demo/demo-esm-webworker.js b/demo/demo-esm-webworker.js index 6f3714c0..982b23f0 100644 --- a/demo/demo-esm-webworker.js +++ b/demo/demo-esm-webworker.js @@ -8,10 +8,13 @@ const log = (...msg) => { }; onmessage = async (msg) => { + // worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]); + const image = new ImageData(new Uint8ClampedArray(msg.data.image), msg.data.width, msg.data.height); config = msg.data.config; let result = {}; try { - result = await human.detect(msg.data.image, config); + // result = await human.detect(image, config); + result = {}; } catch (err) { result.error = err.message; log('Worker thread error:', err.message); diff --git a/demo/demo-esm.js b/demo/demo-esm.js index 1db51d7c..fa4eac92 100644 --- a/demo/demo-esm.js +++ b/demo/demo-esm.js @@ -17,16 +17,17 @@ const config = { detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, mesh: { enabled: true }, iris: { enabled: true }, - age: { enabled: true, skipFrames: 10 }, - gender: { enabled: true }, - emotion: { enabled: true, minConfidence: 0.5, useGrayscale: true }, + age: { enabled: false, skipFrames: 10 }, + gender: { enabled: false }, + emotion: { enabled: false, minConfidence: 0.5, useGrayscale: true }, }, - body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, - hand: { enabled: true, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, + body: { enabled: false, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 }, + hand: { enabled: false, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, }; let settings; let worker; let timeStamp; +const fps = []; function str(...msg) { if (!Array.isArray(msg)) return msg; @@ -44,6 +45,7 @@ const log = (...msg) => { }; async function drawFace(result, canvas) { + if (!result) return; const ctx = canvas.getContext('2d'); ctx.strokeStyle = ui.baseColor; ctx.font = ui.baseFont; @@ -96,6 +98,7 @@ async function drawFace(result, canvas) { } async function drawBody(result, canvas) { + if (!result) return; const ctx = canvas.getContext('2d'); ctx.fillStyle = ui.baseColor; ctx.strokeStyle = ui.baseColor; @@ -157,6 +160,7 @@ async function drawBody(result, canvas) { } async function drawHand(result, canvas) { + if (!result) return; const ctx = canvas.getContext('2d'); ctx.font = ui.baseFont; ctx.lineWidth = ui.baseLineWidth; @@ -203,6 +207,13 @@ async function drawHand(result, canvas) { async function drawResults(input, result, canvas) { // update fps settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp))); + fps.push(1000 / (performance.now() - timeStamp)); + if (fps.length > 20) fps.shift(); + settings.setValue('FPS', Math.round(10 * fps.reduce((a, b) => a + b) / fps.length) / 10); + + // eslint-disable-next-line no-use-before-define + requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop + // draw image from video const ctx = canvas.getContext('2d'); ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); @@ -213,27 +224,24 @@ async function drawResults(input, result, canvas) { // update log const engine = await human.tf.engine(); const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`; - const gpu = engine.backendInstance.numBytesInGPU ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : ''; + const gpu = engine.backendInstance ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : ''; document.getElementById('log').innerText = ` TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu} Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes `; } -async function webWorker(input, image, canvas) { +// simple wrapper for worker.postmessage that creates worker if one does not exist +function webWorker(input, image, canvas) { if (!worker) { + // create new webworker and add event handler only once log('Creating worker thread'); - // create new webworker worker = new Worker('demo-esm-webworker.js', { type: 'module' }); // after receiving message from webworker, parse&draw results and send new frame for processing - worker.addEventListener('message', async (msg) => { - await drawResults(input, msg.data, canvas); - // eslint-disable-next-line no-use-before-define - requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop - }); + worker.addEventListener('message', async (msg) => drawResults(input, msg.data, canvas)); } - // const offscreen = image.transferControlToOffscreen(); - worker.postMessage({ image, config }); + // pass image data as arraybuffer to worker by reference to avoid copy + worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]); } async function runHumanDetect(input, canvas) { @@ -247,17 +255,17 @@ async function runHumanDetect(input, canvas) { const ctx = offscreen.getContext('2d'); ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); const data = ctx.getImageData(0, 0, canvas.width, canvas.height); - // perform detection - await webWorker(input, data, canvas); + // perform detection in worker + webWorker(input, data, canvas); } else { let result = {}; try { + // perform detection result = await human.detect(input, config); } catch (err) { log('Error during execution:', err.message); } - await drawResults(input, result, canvas); - if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop + drawResults(input, result, canvas); } } }