enhanced processing resolution

demo/browser.js

@@ -69,6 +69,7 @@ let menu;
 let menuFX;
 let worker;
 let timeStamp;
+let camera = {};
 const fps = [];
 
 // helper function: translates json to human readable string
@@ -111,19 +112,26 @@ function drawResults(input, result, canvas) {
   const ctx = canvas.getContext('2d');
   ctx.fillStyle = ui.baseBackground;
   ctx.fillRect(0, 0, canvas.width, canvas.height);
-  if (result.canvas) ctx.drawImage(result.canvas, 0, 0, result.canvas.width, result.canvas.height, 0, 0, result.canvas.width, result.canvas.height);
+  if (result.canvas) {
-  else ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
+    if (result.canvas.width !== canvas.width) canvas.width = result.canvas.width;
+    if (result.canvas.height !== canvas.height) canvas.height = result.canvas.height;
+    ctx.drawImage(result.canvas, 0, 0, result.canvas.width, result.canvas.height, 0, 0, result.canvas.width, result.canvas.height);
+  } else {
+    ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
+  }
   // draw all results
   draw.face(result.face, canvas, ui, human.facemesh.triangulation);
   draw.body(result.body, canvas, ui);
   draw.hand(result.hand, canvas, ui);
   // update log
   const engine = human.tf.engine();
-  const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
+  const gpu = engine.backendInstance ? `gpu: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : '';
-  const gpu = engine.backendInstance ? `GPU: ${(engine.backendInstance.numBytesInGPU ? engine.backendInstance.numBytesInGPU : 0).toLocaleString()} bytes` : '';
+  const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} tensors: ${engine.state.numTensors.toLocaleString()}`;
+  const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : '';
   document.getElementById('log').innerText = `
-    TFJS Version: ${human.tf.version_core} | Backend: ${human.tf.getBackend()} | Memory: ${memory} ${gpu}
+    video: ${camera.name} facing: ${camera.facing} resolution: ${camera.width} x ${camera.height} ${processing}
-    Performance: ${str(result.performance)} | Object size: ${(str(result)).length.toLocaleString()} bytes
+    backend: ${human.tf.getBackend()} | ${memory} | object size: ${(str(result)).length.toLocaleString()} bytes
+    performance: ${str(result.performance)}
   `;
 }
 
@@ -151,7 +159,7 @@ async function setupCamera() {
   try {
     stream = await navigator.mediaDevices.getUserMedia({
       audio: false,
-      video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight },
+      video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight, resizeMode: 'none' },
     });
   } catch (err) {
     output.innerText += '\nCamera permission denied';
@@ -160,6 +168,10 @@ async function setupCamera() {
   }
   if (stream) video.srcObject = stream;
   else return null;
+  const track = stream.getVideoTracks()[0];
+  const settings = track.getSettings();
+  log('camera settings:', settings);
+  camera = { name: track.label, width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
   return new Promise((resolve) => {
     video.onloadeddata = async () => {
       video.width = video.videoWidth;
@@ -169,8 +181,7 @@ async function setupCamera() {
       if (live) video.play();
       ui.busy = false;
       // do once more because onresize events can be delayed or skipped
-      if (video.width > window.innerWidth) await setupCamera();
+      // if (video.width > window.innerWidth) await setupCamera();
-      output.innerText += `\nCamera resolution: ${video.width} x ${video.height}`;
       resolve(video);
     };
   });
@@ -350,8 +361,8 @@ function setupMenu() {
   menuFX.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
   menuFX.addLabel('Image Filters');
   menuFX.addBool('Enabled', config.filter, 'enabled');
-  menuFX.addRange('Image width', config.filter, 'width', 100, 3840, 10, (val) => config.filter.width = parseInt(val));
+  menuFX.addRange('Image width', config.filter, 'width', 0, 3840, 10, (val) => config.filter.width = parseInt(val));
-  menuFX.addRange('Image height', config.filter, 'height', 100, 2160, 10, (val) => config.filter.height = parseInt(val));
+  menuFX.addRange('Image height', config.filter, 'height', 0, 2160, 10, (val) => config.filter.height = parseInt(val));
   menuFX.addRange('Brightness', config.filter, 'brightness', -1.0, 1.0, 0.05, (val) => config.filter.brightness = parseFloat(val));
   menuFX.addRange('Contrast', config.filter, 'contrast', -1.0, 1.0, 0.05, (val) => config.filter.contrast = parseFloat(val));
   menuFX.addRange('Sharpness', config.filter, 'sharpness', 0, 1.0, 0.05, (val) => config.filter.sharpness = parseFloat(val));

dist/human.esm-nobundle.json

@@ -9,7 +9,7 @@
       "imports": []
     },
     "src/emotion/emotion.js": {
-      "bytes": 1954,
+      "bytes": 1976,
       "imports": [
         {
           "path": "src/profile.js"
@@ -127,7 +127,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 15055,
+      "bytes": 15164,
       "imports": [
         {
           "path": "src/facemesh/facemesh.js"
@@ -282,7 +282,7 @@
     "dist/human.esm-nobundle.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 605105
+      "bytes": 605307
     },
     "dist/human.esm-nobundle.js": {
       "imports": [],
@@ -318,7 +318,7 @@
           "bytesInOutput": 1236
         },
         "src/emotion/emotion.js": {
-          "bytesInOutput": 1082
+          "bytesInOutput": 1104
         },
         "src/posenet/modelBase.js": {
           "bytesInOutput": 455
@@ -384,13 +384,13 @@
           "bytesInOutput": 2567
         },
         "src/human.js": {
-          "bytesInOutput": 8359
+          "bytesInOutput": 8439
         },
         "src/human.js": {
           "bytesInOutput": 0
         }
       },
-      "bytes": 211512
+      "bytes": 211614
     }
   }
 }

dist/human.esm.json

@@ -153,7 +153,7 @@
       "imports": []
     },
     "src/emotion/emotion.js": {
-      "bytes": 1954,
+      "bytes": 1976,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -301,7 +301,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 15055,
+      "bytes": 15164,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -488,7 +488,7 @@
     "dist/human.esm.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5400887
+      "bytes": 5401089
     },
     "dist/human.esm.js": {
       "imports": [],
@@ -581,7 +581,7 @@
           "bytesInOutput": 1236
         },
         "src/emotion/emotion.js": {
-          "bytesInOutput": 1071
+          "bytesInOutput": 1093
         },
         "src/posenet/modelBase.js": {
           "bytesInOutput": 433
@@ -647,13 +647,13 @@
           "bytesInOutput": 2568
         },
         "src/human.js": {
-          "bytesInOutput": 8375
+          "bytesInOutput": 8455
         },
         "src/human.js": {
           "bytesInOutput": 0
         }
       },
-      "bytes": 1273058
+      "bytes": 1273160
     }
   }
 }

dist/human.json

@@ -153,7 +153,7 @@
       "imports": []
     },
     "src/emotion/emotion.js": {
-      "bytes": 1954,
+      "bytes": 1976,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -301,7 +301,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 15055,
+      "bytes": 15164,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -488,7 +488,7 @@
     "dist/human.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5400883
+      "bytes": 5401085
     },
     "dist/human.js": {
       "imports": [],
@@ -581,7 +581,7 @@
           "bytesInOutput": 1236
         },
         "src/emotion/emotion.js": {
-          "bytesInOutput": 1071
+          "bytesInOutput": 1093
         },
         "src/posenet/modelBase.js": {
           "bytesInOutput": 433
@@ -647,10 +647,10 @@
           "bytesInOutput": 2567
         },
         "src/human.js": {
-          "bytesInOutput": 8413
+          "bytesInOutput": 8493
         }
       },
-      "bytes": 1273103
+      "bytes": 1273205
     }
   }
 }

dist/human.node.json

@@ -9,7 +9,7 @@
       "imports": []
     },
     "src/emotion/emotion.js": {
-      "bytes": 1954,
+      "bytes": 1976,
       "imports": [
         {
           "path": "src/profile.js"
@@ -127,7 +127,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 15055,
+      "bytes": 15164,
       "imports": [
         {
           "path": "src/facemesh/facemesh.js"
@@ -282,7 +282,7 @@
     "dist/human.node-nobundle.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 620560
+      "bytes": 620872
     },
     "dist/human.node-nobundle.js": {
       "imports": [],
@@ -318,7 +318,7 @@
           "bytesInOutput": 1236
         },
         "src/emotion/emotion.js": {
-          "bytesInOutput": 1082
+          "bytesInOutput": 1104
         },
         "src/posenet/modelBase.js": {
           "bytesInOutput": 455
@@ -387,10 +387,10 @@
           "bytesInOutput": 29
         },
         "src/human.js": {
-          "bytesInOutput": 8359
+          "bytesInOutput": 8439
         }
       },
-      "bytes": 211520
+      "bytes": 211622
     }
   }
 }

src/emotion/emotion.js

@@ -4,7 +4,7 @@ const profile = require('../profile.js');
 const annotations = ['angry', 'discust', 'fear', 'happy', 'sad', 'surpise', 'neutral'];
 const models = {};
 let last = [];
-let frame = 0;
+let frame = Number.MAX_SAFE_INTEGER;
 const multiplier = 1.5;
 
 async function load(config) {

src/human.js

@@ -9,8 +9,6 @@ const profile = require('./profile.js');
 const defaults = require('../config.js').default;
 const app = require('../package.json');
 
-let first = true;
-
 // static config override for non-video detection
 const override = {
   face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } },
@@ -42,19 +40,6 @@ function mergeDeep(...objects) {
   }, {});
 }
 
-function sanity(input) {
-  if (!input) return 'input is not defined';
-  if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
-    return 'input must be a tensor';
-  }
-  try {
-    tf.getBackend();
-  } catch {
-    return 'backend not loaded';
-  }
-  return null;
-}
-
 class Human {
   constructor() {
     this.tf = tf;
@@ -65,6 +50,8 @@ class Human {
     this.state = 'idle';
     this.numTensors = 0;
     this.analyzeMemoryLeaks = false;
+    this.checkSanity = false;
+    this.firstRun = true;
     // internal temp canvases
     this.inCanvas = null;
     this.outCanvas = null;
@@ -107,14 +94,28 @@ class Human {
     if (leaked !== 0) this.log(...msg, leaked);
   }
 
+  sanity(input) {
+    if (!this.checkSanity) return null;
+    if (!input) return 'input is not defined';
+    if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
+      return 'input must be a tensor';
+    }
+    try {
+      tf.getBackend();
+    } catch {
+      return 'backend not loaded';
+    }
+    return null;
+  }
+
   async load(userConfig) {
     if (userConfig) this.config = mergeDeep(defaults, userConfig);
 
-    if (first) {
+    if (this.firstRun) {
       this.log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`);
       this.log('configuration:', this.config);
       this.log('flags:', tf.ENV.flags);
-      first = false;
+      this.firstRun = false;
     }
 
     if (this.config.face.enabled && !this.models.facemesh) {
@@ -183,7 +184,7 @@ class Human {
       else if (this.config.filter.height > 0) targetWidth = originalWidth * (this.config.filter.height / originalHeight);
       if (this.config.filter.height > 0) targetHeight = this.config.filter.height;
       else if (this.config.filter.width > 0) targetHeight = originalHeight * (this.config.filter.width / originalWidth);
-      if (!this.inCanvas || (this.inCanvas.width !== originalWidth) || (this.inCanvas.height !== originalHeight)) {
+      if (!this.inCanvas || (this.inCanvas.width !== targetWidth) || (this.inCanvas.height !== targetHeight)) {
         this.inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
         if (this.inCanvas.width !== targetWidth) this.inCanvas.width = targetWidth;
         if (this.inCanvas.height !== targetHeight) this.inCanvas.height = targetHeight;
@@ -248,7 +249,7 @@ class Human {
 
     // sanity checks
     this.state = 'check';
-    const error = sanity(input);
+    const error = this.sanity(input);
     if (error) {
       this.log(error, input);
       return { error };