From fcdb076549f3feb65daa75daedc6b09b8974342a Mon Sep 17 00:00:00 2001
From: Vladimir Mandic <mandic00@live.com>
Date: Wed, 11 Nov 2020 15:02:49 -0500
Subject: [PATCH] added performance notes

---
 demo/browser.js | 10 ++++++----
 package.json    |  3 +--
 src/human.js    | 12 ++++++------
 src/image.js    | 26 ++++++++++++++++++++++++--
 wiki            |  2 +-
 5 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/demo/browser.js b/demo/browser.js
index 228598fa..eec18f52 100644
--- a/demo/browser.js
+++ b/demo/browser.js
@@ -105,10 +105,12 @@ function drawResults(input, result, canvas) {
   const memory = `system: ${engine.state.numBytes.toLocaleString()} bytes ${gpu} | tensors: ${engine.state.numTensors.toLocaleString()}`;
   const processing = result.canvas ? `processing: ${result.canvas.width} x ${result.canvas.height}` : '';
   const avg = Math.trunc(10 * ui.fps.reduce((a, b) => a + b) / ui.fps.length) / 10;
-  document.getElementById('log').innerText = `
-    video: ${ui.camera.name} | facing: ${ui.camera.facing} | resolution: ${ui.camera.width} x ${ui.camera.height} ${processing}
-    backend: ${human.tf.getBackend()} | ${memory}
-    performance: ${str(result.performance)} FPS:${avg}
+  const warning = (ui.fps.length > 5) && (avg < 5) ? '<font color="lightcoral">warning: your performance is low: try switching to higher performance backend, lowering resolution or disabling some models</font>' : '';
+  document.getElementById('log').innerHTML = `
+    video: ${ui.camera.name} | facing: ${ui.camera.facing} | resolution: ${ui.camera.width} x ${ui.camera.height} ${processing}<br>
+    backend: ${human.tf.getBackend()} | ${memory}<br>
+    performance: ${str(result.performance)} FPS:${avg}<br>
+    ${warning}
   `;
 }
 
diff --git a/package.json b/package.json
index 55f1085a..91f4ea09 100644
--- a/package.json
+++ b/package.json
@@ -19,8 +19,7 @@
     "type": "git",
     "url": "git+https://github.com/vladmandic/human.git"
   },
-  "dependencies": {
-  },
+  "dependencies": {},
   "peerDependencies": {},
   "devDependencies": {
     "@tensorflow/tfjs": "^2.7.0",
diff --git a/src/human.js b/src/human.js
index 1134a574..7d78a356 100644
--- a/src/human.js
+++ b/src/human.js
@@ -130,12 +130,12 @@ class Human {
         this.models.posenet,
         this.models.handpose,
       ] = await Promise.all([
-        this.config.face.age.enabled ? this.models.age || age.load(this.config) : null,
-        this.config.face.gender.enabled ? this.models.gender || gender.load(this.config) : null,
-        this.config.face.emotion.enabled ? this.models.emotion || emotion.load(this.config) : null,
-        this.config.face.enabled ? this.models.facemesh || facemesh.load(this.config.face) : null,
-        this.config.body.enabled ? this.models.posenet || posenet.load(this.config) : null,
-        this.config.hand.enabled ? this.models.handpose || handpose.load(this.config.hand) : null,
+        this.models.age || (this.config.face.age.enabled ? age.load(this.config) : null),
+        this.models.gender || (this.config.face.gender.enabled ? gender.load(this.config) : null),
+        this.models.emotion || (this.config.face.emotion.enabled ? emotion.load(this.config) : null),
+        this.models.facemesh || (this.config.face.enabled ? facemesh.load(this.config.face) : null),
+        this.models.posenet || (this.config.body.enabled ? posenet.load(this.config) : null),
+        this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config.hand) : null),
       ]);
     } else {
       if (this.config.face.enabled && !this.models.facemesh) this.models.facemesh = await facemesh.load(this.config.face);
diff --git a/src/image.js b/src/image.js
index 97a086c7..f0964755 100644
--- a/src/image.js
+++ b/src/image.js
@@ -52,12 +52,34 @@ function process(input, config) {
       if (config.filter.polaroid) this.fx.addFilter('polaroid');
       if (config.filter.pixelate !== 0) this.fx.addFilter('pixelate', config.filter.pixelate);
       this.fx.apply(inCanvas);
+      // read pixel data
+      // const gl = outCanvas.getContext('webgl');
+      const gl = false;
+      if (gl) {
+        const glBuffer = new Uint8Array(outCanvas.width * outCanvas.height * 4);
+        const pixBuffer = new Uint8Array(outCanvas.width * outCanvas.height * 3);
+        gl.readPixels(0, 0, outCanvas.width, outCanvas.height, gl.RGBA, gl.UNSIGNED_BYTE, glBuffer);
+        // gl returns rbga while we only need rgb, so discarding alpha channel
+        // gl returns starting point as lower left, so need to invert vertical
+        let i = 0;
+        for (let y = outCanvas.height - 1; y >= 0; y--) {
+          for (let x = 0; x < outCanvas.width; x++) {
+            const index = (x + y * outCanvas.width) * 4;
+            pixBuffer[i++] = glBuffer[index + 0];
+            pixBuffer[i++] = glBuffer[index + 1];
+            pixBuffer[i++] = glBuffer[index + 2];
+          }
+        }
+        outCanvas.data = pixBuffer;
+      }
     } else {
       outCanvas = inCanvas;
     }
-    // if (!outCanvas) outCanvas = inCanvas;
     let pixels;
-    if ((config.backend === 'webgl') || (outCanvas instanceof ImageData)) {
+    if (outCanvas.data) {
+      const shape = [outCanvas.height, outCanvas.width, 3];
+      pixels = tf.tensor3d(outCanvas.data, shape, 'int32');
+    } else if ((config.backend === 'webgl') || (outCanvas instanceof ImageData)) {
       // tf kernel-optimized method to get imagedata, also if input is imagedata, just use it
       pixels = tf.browser.fromPixels(outCanvas);
     } else {
diff --git a/wiki b/wiki
index ca368866..6b460e9f 160000
--- a/wiki
+++ b/wiki
@@ -1 +1 @@
-Subproject commit ca3688663e1c427fdd26f55bde54336e939dcc5f
+Subproject commit 6b460e9f5252038ef7a94b044fdb789e35d610bd