implemented blink detection

README.md

@@ -22,6 +22,7 @@
 - [**Usage**](https://github.com/vladmandic/human/wiki/Usage)
 - [**Configuration**](https://github.com/vladmandic/human/wiki/Configuration)
 - [**Outputs**](https://github.com/vladmandic/human/wiki/Outputs)
+- [**Gesture Recognition**](https://github.com/vladmandic/human/wiki/Gesture)
 - [**Demos**](https://github.com/vladmandic/human/wiki/Demos)
 - [**Development Server**](https://github.com/vladmandic/human/wiki/Development-Server)
 - [**Build Process**](https://github.com/vladmandic/human/wiki/Build-Process)

demo/browser.js

@@ -297,7 +297,7 @@ async function detectSampleImages() {
 }
 
 function setupMenu() {
-  menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' });
+  menu = new Menu(document.body, '', { top: '1rem', right: '1rem' });
   const btn = menu.addButton('start video', 'pause video', () => detectVideo());
   menu.addButton('process images', 'process images', () => detectSampleImages());
   document.getElementById('play').addEventListener('click', () => btn.click());
@@ -352,7 +352,7 @@ function setupMenu() {
   menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
   menu.addChart('FPS', 'FPS');
 
-  menuFX = new Menu(document.body, '...', { top: '1rem', right: '18rem' });
+  menuFX = new Menu(document.body, '', { top: '1rem', right: '18rem' });
   menuFX.addLabel('ui options');
   menuFX.addBool('crop & scale', ui, 'crop', () => setupCamera());
   menuFX.addBool('camera front/back', ui, 'facing', () => setupCamera());

demo/draw.js

@@ -68,6 +68,31 @@ async function drawFace(result, canvas, ui, triangulation) {
             ctx.fill(path);
           }
         }
+        // iris: array[center, left, top, right, bottom]
+        if (face.annotations && face.annotations.leftEyeIris) {
+          ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
+          ctx.beginPath();
+          const sizeX = Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]) / 2;
+          const sizeY = Math.abs(face.annotations.leftEyeIris[4][1] - face.annotations.leftEyeIris[2][1]) / 2;
+          ctx.ellipse(face.annotations.leftEyeIris[0][0], face.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
+          ctx.stroke();
+          if (ui.fillPolygons) {
+            ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
+            ctx.fill();
+          }
+        }
+        if (face.annotations && face.annotations.rightEyeIris) {
+          ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
+          ctx.beginPath();
+          const sizeX = Math.abs(face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) / 2;
+          const sizeY = Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]) / 2;
+          ctx.ellipse(face.annotations.rightEyeIris[0][0], face.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
+          ctx.stroke();
+          if (ui.fillPolygons) {
+            ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
+            ctx.fill();
+          }
+        }
       }
     }
   }

demo/menu.js

@@ -55,6 +55,9 @@ function createCSS() {
   input[type=range] { -webkit-appearance: none; }
   input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; }
   input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; }
+
+  .svg-background { fill:darkslategrey; cursor:pointer; opacity: 0.6; }
+  .svg-foreground { fill:white; cursor:pointer; opacity: 0.8; }
   `;
   const el = document.createElement('style');
   el.innerHTML = css;
@@ -89,18 +92,22 @@ class Menu {
     this.container.id = `menu-container-${instance}`;
     this.container.className = 'menu-container menu-container-fadein';
 
-    if (title !== '') {
+    // set menu title with pulldown arrow
     const elTitle = document.createElement('div');
     elTitle.className = 'menu-title';
     elTitle.id = `menu-title-${instance}`;
-      elTitle.innerHTML = title;
+    const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" style="width: 2rem; height: 2rem; vertical-align: top;">
+        <path d="M400 32H48A48 48 0 0 0 0 80v352a48 48 0 0 0 48 48h352a48 48 0 0 0 48-48V80a48 48 0 0 0-48-48zm-51.37 182.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-background"/>
+        <path d="M348.63 214.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-foreground"/>
+      </svg>`;
+    elTitle.innerHTML = `${title}${svg}`;
     this.menu.appendChild(elTitle);
     elTitle.addEventListener('click', () => {
       this.container.classList.toggle('menu-container-fadeout');
       this.container.classList.toggle('menu-container-fadein');
       this.menu.style.borderStyle = this.container.classList.contains('menu-container-fadeout') ? 'none' : 'solid';
     });
-    }
+
     this.menu.appendChild(this.container);
     if (typeof parent === 'object') parent.appendChild(this.menu);
     else document.getElementById(parent).appendChild(this.menu);

dist/demo-browser-index.js

@@ -32958,8 +32958,13 @@ var rL = we((pc) => {
     if (!n)
       return [];
     const t = [];
-    for (const e of n)
+    for (const e of n) {
       e.annotations.rightCheek && e.annotations.leftCheek && e.annotations.rightCheek.length > 0 && e.annotations.leftCheek.length > 0 && t.push(`facing ${e.annotations.rightCheek[0][2] > 0 || e.annotations.leftCheek[0][2] < 0 ? "right" : "left"}`);
+      const r = Math.abs(e.mesh[374][1] - e.mesh[386][1]) / Math.abs(e.mesh[443][1] - e.mesh[450][1]);
+      r < 0.2 && t.push("blink left eye");
+      const i = Math.abs(e.mesh[145][1] - e.mesh[159][1]) / Math.abs(e.mesh[223][1] - e.mesh[230][1]);
+      i < 0.2 && t.push("blink right eye");
+    }
     return t;
   };
   pc.hand = (n) => {
@@ -33236,8 +33241,8 @@ class hG {
         continue;
       }
       this.analyze("Start Age:"), this.config.async ? e = this.config.face.age.enabled ? xs.predict(o.image, this.config) : {} : (this.state = "run:age", t = ut(), e = this.config.face.age.enabled ? await xs.predict(o.image, this.config) : {}, this.perf.age = Math.trunc(ut() - t)), this.analyze("Start Gender:"), this.config.async ? r = this.config.face.gender.enabled ? Ls.predict(o.image, this.config) : {} : (this.state = "run:gender", t = ut(), r = this.config.face.gender.enabled ? await Ls.predict(o.image, this.config) : {}, this.perf.gender = Math.trunc(ut() - t)), this.analyze("Start Emotion:"), this.config.async ? i = this.config.face.emotion.enabled ? Ss.predict(o.image, this.config) : {} : (this.state = "run:emotion", t = ut(), i = this.config.face.emotion.enabled ? await Ss.predict(o.image, this.config) : {}, this.perf.emotion = Math.trunc(ut() - t)), this.analyze("End Emotion:"), this.config.async && ([e, r, i] = await Promise.all([e, r, i])), this.analyze("Finish Face:"), o.image.dispose();
-      const c = o.annotations.leftEyeIris && o.annotations.rightEyeIris ? Math.max(o.annotations.leftEyeIris[3][0] - o.annotations.leftEyeIris[1][0], o.annotations.rightEyeIris[3][0] - o.annotations.rightEyeIris[1][0]) : 0;
+      const c = o.annotations.leftEyeIris && o.annotations.rightEyeIris ? 11.7 * Math.max(Math.abs(o.annotations.leftEyeIris[3][0] - o.annotations.leftEyeIris[1][0]), Math.abs(o.annotations.rightEyeIris[4][1] - o.annotations.rightEyeIris[2][1])) : 0;
-      a.push({confidence: o.confidence, box: o.box, mesh: o.mesh, annotations: o.annotations, age: e.age, gender: r.gender, genderConfidence: r.confidence, emotion: i, iris: c !== 0 ? Math.trunc(100 * 11.7 / c) / 100 : 0}), this.analyze("End Face");
+      a.push({confidence: o.confidence, box: o.box, mesh: o.mesh, annotations: o.annotations, age: e.age, gender: r.gender, genderConfidence: r.confidence, emotion: i, iris: c !== 0 ? Math.trunc(c) / 100 : 0}), this.analyze("End Face");
     }
     return this.analyze("End FaceMesh:"), this.config.async && (this.perf.face && delete this.perf.face, this.perf.age && delete this.perf.age, this.perf.gender && delete this.perf.gender, this.perf.emotion && delete this.perf.emotion), a;
   }
@@ -33536,6 +33541,30 @@ async function drawFace(result, canvas, ui2, triangulation) {
             ctx.fill(path);
           }
         }
+        if (face.annotations && face.annotations.leftEyeIris) {
+          ctx.strokeStyle = ui2.useDepth ? "rgba(255, 200, 255, 0.3)" : ui2.baseColor;
+          ctx.beginPath();
+          const sizeX = Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]) / 2;
+          const sizeY = Math.abs(face.annotations.leftEyeIris[4][1] - face.annotations.leftEyeIris[2][1]) / 2;
+          ctx.ellipse(face.annotations.leftEyeIris[0][0], face.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
+          ctx.stroke();
+          if (ui2.fillPolygons) {
+            ctx.fillStyle = ui2.useDepth ? "rgba(255, 255, 200, 0.3)" : ui2.baseColor;
+            ctx.fill();
+          }
+        }
+        if (face.annotations && face.annotations.rightEyeIris) {
+          ctx.strokeStyle = ui2.useDepth ? "rgba(255, 200, 255, 0.3)" : ui2.baseColor;
+          ctx.beginPath();
+          const sizeX = Math.abs(face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) / 2;
+          const sizeY = Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]) / 2;
+          ctx.ellipse(face.annotations.rightEyeIris[0][0], face.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
+          ctx.stroke();
+          if (ui2.fillPolygons) {
+            ctx.fillStyle = ui2.useDepth ? "rgba(255, 255, 200, 0.3)" : ui2.baseColor;
+            ctx.fill();
+          }
+        }
       }
     }
   }
@@ -33716,6 +33745,9 @@ function createCSS() {
   input[type=range] { -webkit-appearance: none; }
   input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; }
   input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; }
+
+  .svg-background { fill:darkslategrey; cursor:pointer; opacity: 0.6; }
+  .svg-foreground { fill:white; cursor:pointer; opacity: 0.8; }
   `;
   const el = document.createElement("style");
   el.innerHTML = css;
@@ -33751,18 +33783,20 @@ class Menu {
     this.container = document.createElement("div");
     this.container.id = `menu-container-${instance}`;
     this.container.className = "menu-container menu-container-fadein";
-    if (title !== "") {
     const elTitle = document.createElement("div");
     elTitle.className = "menu-title";
     elTitle.id = `menu-title-${instance}`;
-      elTitle.innerHTML = title;
+    const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" style="width: 2rem; height: 2rem; vertical-align: top;">
+        <path d="M400 32H48A48 48 0 0 0 0 80v352a48 48 0 0 0 48 48h352a48 48 0 0 0 48-48V80a48 48 0 0 0-48-48zm-51.37 182.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-background"/>
+        <path d="M348.63 214.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-foreground"/>
+      </svg>`;
+    elTitle.innerHTML = `${title}${svg}`;
     this.menu.appendChild(elTitle);
     elTitle.addEventListener("click", () => {
       this.container.classList.toggle("menu-container-fadeout");
       this.container.classList.toggle("menu-container-fadein");
       this.menu.style.borderStyle = this.container.classList.contains("menu-container-fadeout") ? "none" : "solid";
     });
-    }
     this.menu.appendChild(this.container);
     if (typeof parent === "object")
       parent.appendChild(this.menu);
@@ -34228,7 +34262,7 @@ async function detectSampleImages() {
   status("");
 }
 function setupMenu() {
-  menu2 = new menu_default(document.body, "...", {top: "1rem", right: "1rem"});
+  menu2 = new menu_default(document.body, "", {top: "1rem", right: "1rem"});
   const btn = menu2.addButton("start video", "pause video", () => detectVideo());
   menu2.addButton("process images", "process images", () => detectSampleImages());
   document.getElementById("play").addEventListener("click", () => btn.click());
@@ -34279,7 +34313,7 @@ function setupMenu() {
   });
   menu2.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
   menu2.addChart("FPS", "FPS");
-  menuFX = new menu_default(document.body, "...", {top: "1rem", right: "18rem"});
+  menuFX = new menu_default(document.body, "", {top: "1rem", right: "18rem"});
   menuFX.addLabel("ui options");
   menuFX.addBool("crop & scale", ui, "crop", () => setupCamera());
   menuFX.addBool("camera front/back", ui, "facing", () => setupCamera());

dist/demo-browser-index.json

@@ -1,7 +1,7 @@
 {
   "inputs": {
     "demo/browser.js": {
-      "bytes": 18569,
+      "bytes": 18563,
       "imports": [
         {
           "path": "dist/human.esm.js"
@@ -15,15 +15,15 @@
       ]
     },
     "demo/draw.js": {
-      "bytes": 7627,
+      "bytes": 9119,
       "imports": []
     },
     "demo/menu.js": {
-      "bytes": 12725,
+      "bytes": 13466,
       "imports": []
     },
     "dist/human.esm.js": {
-      "bytes": 1278262,
+      "bytes": 1278518,
       "imports": []
     }
   },
@@ -31,28 +31,28 @@
     "dist/demo-browser-index.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5530763
+      "bytes": 5534714
     },
     "dist/demo-browser-index.js": {
       "imports": [],
       "inputs": {
         "dist/human.esm.js": {
-          "bytesInOutput": 1664653
+          "bytesInOutput": 1664968
         },
         "dist/human.esm.js": {
           "bytesInOutput": 8716
         },
         "demo/draw.js": {
-          "bytesInOutput": 7451
+          "bytesInOutput": 8896
         },
         "demo/menu.js": {
-          "bytesInOutput": 12727
+          "bytesInOutput": 13425
         },
         "demo/browser.js": {
-          "bytesInOutput": 16215
+          "bytesInOutput": 16209
         }
       },
-      "bytes": 1709884
+      "bytes": 1712336
     }
   }
 }

dist/human.esm-nobundle.json

@@ -194,7 +194,7 @@
       ]
     },
     "src/gesture.js": {
-      "bytes": 2175,
+      "bytes": 2680,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -243,7 +243,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 13725,
+      "bytes": 13770,
       "imports": [
         {
           "path": "src/face/facemesh.js"
@@ -301,7 +301,7 @@
     "dist/human.esm-nobundle.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 620133
+      "bytes": 620981
     },
     "dist/human.esm-nobundle.js": {
       "imports": [],
@@ -394,7 +394,7 @@
           "bytesInOutput": 1219
         },
         "src/gesture.js": {
-          "bytesInOutput": 1219
+          "bytesInOutput": 1459
         },
         "src/imagefx.js": {
           "bytesInOutput": 11013
@@ -409,13 +409,13 @@
           "bytesInOutput": 3047
         },
         "src/human.js": {
-          "bytesInOutput": 7240
+          "bytesInOutput": 7256
         },
         "src/human.js": {
           "bytesInOutput": 0
         }
       },
-      "bytes": 216588
+      "bytes": 216844
     }
   }
 }

dist/human.esm.json

@@ -371,7 +371,7 @@
       ]
     },
     "src/gesture.js": {
-      "bytes": 2175,
+      "bytes": 2680,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -433,7 +433,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 13725,
+      "bytes": 13770,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -513,7 +513,7 @@
     "dist/human.esm.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5416118
+      "bytes": 5416966
     },
     "dist/human.esm.js": {
       "imports": [],
@@ -663,7 +663,7 @@
           "bytesInOutput": 1197
         },
         "src/gesture.js": {
-          "bytesInOutput": 1220
+          "bytesInOutput": 1460
         },
         "src/imagefx.js": {
           "bytesInOutput": 11014
@@ -678,13 +678,13 @@
           "bytesInOutput": 3048
         },
         "src/human.js": {
-          "bytesInOutput": 7258
+          "bytesInOutput": 7274
         },
         "src/human.js": {
           "bytesInOutput": 0
         }
       },
-      "bytes": 1278262
+      "bytes": 1278518
     }
   }
 }

dist/human.json

@@ -371,7 +371,7 @@
       ]
     },
     "src/gesture.js": {
-      "bytes": 2175,
+      "bytes": 2680,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -433,7 +433,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 13725,
+      "bytes": 13770,
       "imports": [
         {
           "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@@ -513,7 +513,7 @@
     "dist/human.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 5416114
+      "bytes": 5416962
     },
     "dist/human.js": {
       "imports": [],
@@ -663,7 +663,7 @@
           "bytesInOutput": 1197
         },
         "src/gesture.js": {
-          "bytesInOutput": 1220
+          "bytesInOutput": 1460
         },
         "src/imagefx.js": {
           "bytesInOutput": 11014
@@ -678,10 +678,10 @@
           "bytesInOutput": 3047
         },
         "src/human.js": {
-          "bytesInOutput": 7296
+          "bytesInOutput": 7312
         }
       },
-      "bytes": 1278307
+      "bytes": 1278563
     }
   }
 }

dist/human.node.json

@@ -194,7 +194,7 @@
       ]
     },
     "src/gesture.js": {
-      "bytes": 2175,
+      "bytes": 2680,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -243,7 +243,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 13725,
+      "bytes": 13770,
       "imports": [
         {
           "path": "src/face/facemesh.js"
@@ -301,7 +301,7 @@
     "dist/human.node-nobundle.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 634264
+      "bytes": 635158
     },
     "dist/human.node-nobundle.js": {
       "imports": [],
@@ -394,7 +394,7 @@
           "bytesInOutput": 1219
         },
         "src/gesture.js": {
-          "bytesInOutput": 1219
+          "bytesInOutput": 1459
         },
         "src/imagefx.js": {
           "bytesInOutput": 11013
@@ -412,10 +412,10 @@
           "bytesInOutput": 28
         },
         "src/human.js": {
-          "bytesInOutput": 7240
+          "bytesInOutput": 7256
         }
       },
-      "bytes": 216595
+      "bytes": 216851
     }
   }
 }

src/gesture.js

@@ -25,6 +25,10 @@ exports.face = (res) => {
     if (face.annotations['rightCheek'] && face.annotations['leftCheek'] && (face.annotations['rightCheek'].length > 0) && (face.annotations['leftCheek'].length > 0)) {
       gestures.push(`facing ${((face.annotations['rightCheek'][0][2] > 0) || (face.annotations['leftCheek'][0][2] < 0)) ? 'right' : 'left'}`);
     }
+    const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
+    if (openLeft < 0.2) gestures.push('blink left eye');
+    const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
+    if (openRight < 0.2) gestures.push('blink right eye');
   }
   return gestures;
 };

src/human.js

@@ -245,9 +245,10 @@ class Human {
       face.image.dispose();
 
       // calculate iris distance
-      // iris: array[ bottom, left, top, right, center ]
+      // iris: array[ center, left, top, right, bottom]
-      const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
+      const irisSize = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
-        ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0])
+        /* average human iris size is 11.7mm */
+        ? 11.7 * Math.max(Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]), Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]))
         : 0;
 
       // combine results
@@ -260,7 +261,7 @@ class Human {
         gender: genderRes.gender,
         genderConfidence: genderRes.confidence,
         emotion: emotionRes,
-        iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
+        iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0,
       });
       this.analyze('End Face');
     }

wiki

@@ -1 +1 @@
-Subproject commit b70041ff6b67e01ac2ed521bdd01c43a970ff351
+Subproject commit 63c2347b5256777d5154bd5d502b2a075d3d950d