implement multi-person gestures

demo/draw.js

@@ -4,9 +4,11 @@ async function drawGesture(result, canvas, ui) {
   ctx.font = ui.baseFont;
   ctx.fillStyle = ui.baseLabel;
   let i = 1;
-  for (const [key, val] of Object.entries(result)) {
-    if (val.length > 0) {
-      const label = `${key}: ${val.join(', ')}`;
+  for (const gesture in result) {
+    const [where, what] = Object.entries(result[gesture]);
+    if ((what.length > 1) && (what[1].length > 0)) {
+      const person = where[1] > 0 ? `#${where[1]}` : '';
+      const label = `${where[0]} ${person}: ${what[1]}`;
       ctx.fillStyle = 'black';
       ctx.fillText(label, 8, 2 + (i * ui.baseLineHeight));
       ctx.fillStyle = ui.baseLabel;

dist/demo-browser-index.json

@@ -18,7 +18,7 @@
       ]
     },
     "demo/draw.js": {
-      "bytes": 10436,
+      "bytes": 10568,
       "imports": []
     },
     "demo/gl-bench.js": {
@@ -30,7 +30,7 @@
       "imports": []
     },
     "dist/human.esm.js": {
-      "bytes": 1784266,
+      "bytes": 1784464,
       "imports": []
     }
   },
@@ -38,17 +38,17 @@
     "dist/demo-browser-index.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2678448
+      "bytes": 2679237
     },
     "dist/demo-browser-index.js": {
       "imports": [],
       "exports": [],
       "inputs": {
         "dist/human.esm.js": {
-          "bytesInOutput": 1777021
+          "bytesInOutput": 1777219
         },
         "demo/draw.js": {
-          "bytesInOutput": 7668
+          "bytesInOutput": 7773
         },
         "demo/menu.js": {
           "bytesInOutput": 11838
@@ -60,7 +60,7 @@
           "bytesInOutput": 19407
         }
       },
-      "bytes": 1830756
+      "bytes": 1831059
     }
   }
 }

dist/human.esm.json

@@ -228,7 +228,7 @@
       ]
     },
     "src/gesture/gesture.js": {
-      "bytes": 2981,
+      "bytes": 3260,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -290,7 +290,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 16057,
+      "bytes": 16046,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -357,7 +357,7 @@
     "dist/human.esm.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2583057
+      "bytes": 2583632
     },
     "dist/human.esm.js": {
       "imports": [],
@@ -444,7 +444,7 @@
           "bytesInOutput": 1929
         },
         "src/gesture/gesture.js": {
-          "bytesInOutput": 2255
+          "bytesInOutput": 2459
         },
         "src/imagefx.js": {
           "bytesInOutput": 13638
@@ -456,7 +456,7 @@
           "bytesInOutput": 1529574
         },
         "src/human.js": {
-          "bytesInOutput": 10604
+          "bytesInOutput": 10598
         },
         "src/hand/box.js": {
           "bytesInOutput": 1880
@@ -471,7 +471,7 @@
           "bytesInOutput": 22
         }
       },
-      "bytes": 1784266
+      "bytes": 1784464
     }
   }
 }

dist/human.json

@@ -228,7 +228,7 @@
       ]
     },
     "src/gesture/gesture.js": {
-      "bytes": 2981,
+      "bytes": 3260,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -290,7 +290,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 16057,
+      "bytes": 16046,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -357,7 +357,7 @@
     "dist/human.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 2550003
+      "bytes": 2550600
     },
     "dist/human.js": {
       "imports": [],
@@ -442,7 +442,7 @@
           "bytesInOutput": 1910
         },
         "src/gesture/gesture.js": {
-          "bytesInOutput": 2235
+          "bytesInOutput": 2459
         },
         "src/imagefx.js": {
           "bytesInOutput": 13638
@@ -451,7 +451,7 @@
           "bytesInOutput": 4044
         },
         "src/human.js": {
-          "bytesInOutput": 10668
+          "bytesInOutput": 10662
         },
         "dist/tfjs.esm.js": {
           "bytesInOutput": 1529065
@@ -469,7 +469,7 @@
           "bytesInOutput": 22
         }
       },
-      "bytes": 1783568
+      "bytes": 1783786
     }
   }
 }

dist/human.node.json

@@ -228,7 +228,7 @@
       ]
     },
     "src/gesture/gesture.js": {
-      "bytes": 2981,
+      "bytes": 3260,
       "imports": []
     },
     "src/hand/anchors.js": {
@@ -290,7 +290,7 @@
       "imports": []
     },
     "src/human.js": {
-      "bytes": 16057,
+      "bytes": 16046,
       "imports": [
         {
           "path": "dist/tfjs.esm.js"
@@ -357,7 +357,7 @@
     "dist/human.node-gpu.js.map": {
       "imports": [],
       "inputs": {},
-      "bytes": 686025
+      "bytes": 686589
     },
     "dist/human.node-gpu.js": {
       "imports": [],
@@ -445,7 +445,7 @@
           "bytesInOutput": 1973
         },
         "src/gesture/gesture.js": {
-          "bytesInOutput": 2259
+          "bytesInOutput": 2463
         },
         "src/imagefx.js": {
           "bytesInOutput": 13620
@@ -454,7 +454,7 @@
           "bytesInOutput": 4107
         },
         "src/human.js": {
-          "bytesInOutput": 10692
+          "bytesInOutput": 10686
         },
         "src/hand/box.js": {
           "bytesInOutput": 1917
@@ -469,7 +469,7 @@
           "bytesInOutput": 21
         }
       },
-      "bytes": 250245
+      "bytes": 250443
     }
   }
 }

src/gesture/gesture.js

@@ -1,19 +1,19 @@
 exports.body = (res) => {
   if (!res) return [];
   const gestures = [];
-  for (const pose of res) {
+  for (const i in res) {
     // raising hands
-    const leftWrist = pose.keypoints.find((a) => (a.part === 'leftWrist'));
-    const rightWrist = pose.keypoints.find((a) => (a.part === 'rightWrist'));
-    const nose = pose.keypoints.find((a) => (a.part === 'nose'));
-    if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push('i give up');
-    else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push('raise left hand');
-    else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push('raise right hand');
+    const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist'));
+    const rightWrist = res[i].keypoints.find((a) => (a.part === 'rightWrist'));
+    const nose = res[i].keypoints.find((a) => (a.part === 'nose'));
+    if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'i give up' });
+    else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise left hand' });
+    else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise right hand' });
 
     // leaning
-    const leftShoulder = pose.keypoints.find((a) => (a.part === 'leftShoulder'));
-    const rightShoulder = pose.keypoints.find((a) => (a.part === 'rightShoulder'));
-    if (leftShoulder && rightShoulder) gestures.push(`leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}`);
+    const leftShoulder = res[i].keypoints.find((a) => (a.part === 'leftShoulder'));
+    const rightShoulder = res[i].keypoints.find((a) => (a.part === 'rightShoulder'));
+    if (leftShoulder && rightShoulder) gestures.push({ body: i, gesture: `leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}` });
   }
   return gestures;
 };
@@ -21,19 +21,19 @@ exports.body = (res) => {
 exports.face = (res) => {
   if (!res) return [];
   const gestures = [];
-  for (const face of res) {
-    if (face.mesh && face.mesh.length > 0) {
-      const eyeFacing = face.mesh[35][2] - face.mesh[263][2];
-      if (Math.abs(eyeFacing) < 10) gestures.push('facing camera');
-      else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`);
-      const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
-      if (openLeft < 0.2) gestures.push('blink left eye');
-      const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
-      if (openRight < 0.2) gestures.push('blink right eye');
-      const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1]));
-      if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`);
-      const chinDepth = face.mesh[152][2];
-      if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`);
+  for (const i in res) {
+    if (res[i].mesh && res[i].mesh.length > 0) {
+      const eyeFacing = res[i].mesh[35][2] - res[i].mesh[263][2];
+      if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing camera' });
+      else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` });
+      const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
+      if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
+      const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
+      if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' });
+      const mouthOpen = Math.min(100, 500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1]) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]));
+      if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` });
+      const chinDepth = res[i].mesh[152][2];
+      if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` });
     }
   }
   return gestures;
@@ -42,15 +42,15 @@ exports.face = (res) => {
 exports.hand = (res) => {
   if (!res) return [];
   const gestures = [];
-  for (const hand of res) {
+  for (const i in res) {
     const fingers = [];
-    for (const [finger, pos] of Object.entries(hand['annotations'])) {
+    for (const [finger, pos] of Object.entries(res[i]['annotations'])) {
       if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger
     }
     if (fingers && fingers.length > 0) {
       const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
       const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
-      gestures.push(`${closest.name} forward ${highest.name} up`);
+      gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` });
     }
   }
   return gestures;

src/human.js

@@ -408,7 +408,7 @@ class Human {
       let gestureRes = [];
       if (this.config.gesture.enabled) {
         timeStamp = now();
-        gestureRes = { face: gesture.face(faceRes), body: gesture.body(poseRes), hand: gesture.hand(handRes) };
+        gestureRes = [...gesture.face(faceRes), ...gesture.body(poseRes), ...gesture.hand(handRes)];
         if (!this.config.async) this.perf.gesture = Math.trunc(now() - timeStamp);
         else if (this.perf.gesture) delete this.perf.gesture;
       }

wiki

@@ -1 +1 @@
-Subproject commit 333457d935d89f2559342b31907d04d091520604
+Subproject commit bcac4981f7df29e367259caf6b3b73e5ecde6519