From 9b0b8cf3900973245a1144d40f1cd8435353eeba Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 3 Nov 2020 10:55:33 -0500 Subject: [PATCH] enhanced processing resolution --- demo/browser.js | 33 ++++++++++++++++++++++----------- src/emotion/emotion.js | 2 +- src/human.js | 39 ++++++++++++++++++++------------------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/demo/browser.js b/demo/browser.js index 196f3516..25ffea41 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -69,6 +69,7 @@ let menu; let menuFX; let worker; let timeStamp; +let camera = {}; const fps = []; // helper function: translates json to human readable string @@ -111,19 +112,26 @@ function drawResults(input, result, canvas) { const ctx = canvas.getContext('2d'); ctx.fillStyle = ui.baseBackground; ctx.fillRect(0, 0, canvas.width, canvas.height); - if (result.canvas) ctx.drawImage(result.canvas, 0, 0, result.canvas.width, result.canvas.height, 0, 0, result.canvas.width, result.canvas.height); - else ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); + if (result.canvas) { + if (result.canvas.width !== canvas.width) canvas.width = result.canvas.width; + if (result.canvas.height !== canvas.height) canvas.height = result.canvas.height; + ctx.drawImage(result.canvas, 0, 0, result.canvas.width, result.canvas.height, 0, 0, result.canvas.width, result.canvas.height); + } else { + ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); + } // draw all results draw.face(result.face, canvas, ui, human.facemesh.triangulation); draw.body(result.body, canvas, ui); draw.hand(result.hand, canvas, ui); // update log const engine = human.tf.engine(); - const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`; - const gpu = engine.backendInstance ? `GPU: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : ''; + const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : ''; + const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} tensors: ${engine.state.numTensors.toLocaleString()}`; + const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : ''; document.getElementById('log').innerText = ` - TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu} - Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes + video: ${camera.name} facing: ${camera.facing} resolution: ${camera.width} x ${camera.height} ${processing} + backend: ${human.tf.getBackend()} | ${memory} | object size: ${(str(result)).length.toLocaleString()} bytes + performance: ${str(result.performance)} `; } @@ -151,7 +159,7 @@ async function setupCamera() { try { stream = await navigator.mediaDevices.getUserMedia({ audio: false, - video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight }, + video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight, resizeMode: 'none' }, }); } catch (err) { output.innerText += '\nCamera permission denied'; @@ -160,6 +168,10 @@ async function setupCamera() { } if (stream) video.srcObject = stream; else return null; + const track = stream.getVideoTracks()[0]; + const settings = track.getSettings(); + log('camera settings:', settings); + camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' }; return new Promise((resolve) => { video.onloadeddata = async () => { video.width = video.videoWidth; @@ -169,8 +181,7 @@ async function setupCamera() { if (live) video.play(); ui.busy = false; // do once more because onresize events can be delayed or skipped - if (video.width > window.innerWidth) await setupCamera(); - output.innerText += `\nCamera resolution: ${video.width} x ${video.height}`; + // if (video.width > window.innerWidth) await setupCamera(); resolve(video); }; }); @@ -350,8 +361,8 @@ function setupMenu() { menuFX.addHTML('
'); menuFX.addLabel('Image Filters'); menuFX.addBool('Enabled', config.filter, 'enabled'); - menuFX.addRange('Image width', config.filter, 'width', 100, 3840, 10, (val) => config.filter.width = parseInt(val)); - menuFX.addRange('Image height', config.filter, 'height', 100, 2160, 10, (val) => config.filter.height = parseInt(val)); + menuFX.addRange('Image width', config.filter, 'width', 0, 3840, 10, (val) => config.filter.width = parseInt(val)); + menuFX.addRange('Image height', config.filter, 'height', 0, 2160, 10, (val) => config.filter.height = parseInt(val)); menuFX.addRange('Brightness', config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => config.filter.brightness = parseFloat(val)); menuFX.addRange('Contrast', config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => config.filter.contrast = parseFloat(val)); menuFX.addRange('Sharpness', config.filter, 'sharpness', 0, 1.0, 0.05, (val) => config.filter.sharpness = parseFloat(val)); diff --git a/src/emotion/emotion.js b/src/emotion/emotion.js index 13112e93..716d9db7 100644 --- a/src/emotion/emotion.js +++ b/src/emotion/emotion.js @@ -4,7 +4,7 @@ const profile = require('../profile.js'); const annotations = ['angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral']; const models = {}; let last = []; -let frame = 0; +let frame = Number.MAX_SAFE_INTEGER; const multiplier = 1.5; async function load(config) { diff --git a/src/human.js b/src/human.js index 37f7bc5f..517f0a0e 100644 --- a/src/human.js +++ b/src/human.js @@ -9,8 +9,6 @@ const profile = require('./profile.js'); const defaults = require('../config.js').default; const app = require('../package.json'); -let first = true; - // static config override for non-video detection const override = { face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } }, @@ -42,19 +40,6 @@ function mergeDeep(...objects) { }, {}); } -function sanity(input) { - if (!input) return 'input is not defined'; - if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) { - return 'input must be a tensor'; - } - try { - tf.getBackend(); - } catch { - return 'backend not loaded'; - } - return null; -} - class Human { constructor() { this.tf = tf; @@ -65,6 +50,8 @@ class Human { this.state = 'idle'; this.numTensors = 0; this.analyzeMemoryLeaks = false; + this.checkSanity = false; + this.firstRun = true; // internal temp canvases this.inCanvas = null; this.outCanvas = null; @@ -107,14 +94,28 @@ class Human { if (leaked !== 0) this.log(...msg, leaked); } + sanity(input) { + if (!this.checkSanity) return null; + if (!input) return 'input is not defined'; + if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) { + return 'input must be a tensor'; + } + try { + tf.getBackend(); + } catch { + return 'backend not loaded'; + } + return null; + } + async load(userConfig) { if (userConfig) this.config = mergeDeep(defaults, userConfig); - if (first) { + if (this.firstRun) { this.log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`); this.log('configuration:', this.config); this.log('flags:', tf.ENV.flags); - first = false; + this.firstRun = false; } if (this.config.face.enabled && !this.models.facemesh) { @@ -183,7 +184,7 @@ class Human { else if (this.config.filter.height > 0) targetWidth = originalWidth * (this.config.filter.height / originalHeight); if (this.config.filter.height > 0) targetHeight = this.config.filter.height; else if (this.config.filter.width > 0) targetHeight = originalHeight * (this.config.filter.width / originalWidth); - if (!this.inCanvas || (this.inCanvas.width !== originalWidth) || (this.inCanvas.height !== originalHeight)) { + if (!this.inCanvas || (this.inCanvas.width !== targetWidth) || (this.inCanvas.height !== targetHeight)) { this.inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); if (this.inCanvas.width !== targetWidth) this.inCanvas.width = targetWidth; if (this.inCanvas.height !== targetHeight) this.inCanvas.height = targetHeight; @@ -248,7 +249,7 @@ class Human { // sanity checks this.state = 'check'; - const error = sanity(input); + const error = this.sanity(input); if (error) { this.log(error, input); return { error };