ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implement multi-person gestures

pull/293/head
Vladimir Mandic 2020-11-23 23:36:04 -05:00
parent 6c6503768a
commit b796b4ea06
3 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/gesture/gesture.js
View File

@ -1,19 +1,19 @@
exports.body = (res) => { exports.body = (res) => {
if (!res) return []; if (!res) return [];
const gestures = []; const gestures = [];
for (const pose of res) { for (const i in res) {
// raising hands // raising hands
const leftWrist = pose.keypoints.find((a) => (a.part === 'leftWrist')); const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist'));
const rightWrist = pose.keypoints.find((a) => (a.part === 'rightWrist')); const rightWrist = res[i].keypoints.find((a) => (a.part === 'rightWrist'));
const nose = pose.keypoints.find((a) => (a.part === 'nose')); const nose = res[i].keypoints.find((a) => (a.part === 'nose'));
if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push('i give up'); if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'i give up' });
else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push('raise left hand'); else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise left hand' });
else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push('raise right hand'); else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise right hand' });
// leaning // leaning
const leftShoulder = pose.keypoints.find((a) => (a.part === 'leftShoulder')); const leftShoulder = res[i].keypoints.find((a) => (a.part === 'leftShoulder'));
const rightShoulder = pose.keypoints.find((a) => (a.part === 'rightShoulder')); const rightShoulder = res[i].keypoints.find((a) => (a.part === 'rightShoulder'));
if (leftShoulder && rightShoulder) gestures.push(`leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}`); if (leftShoulder && rightShoulder) gestures.push({ body: i, gesture: `leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}` });
} }
return gestures; return gestures;
}; };
@ -21,19 +21,19 @@ exports.body = (res) => {
exports.face = (res) => { exports.face = (res) => {
if (!res) return []; if (!res) return [];
const gestures = []; const gestures = [];
for (const face of res) { for (const i in res) {
if (face.mesh && face.mesh.length > 0) { if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = face.mesh[35][2] - face.mesh[263][2]; const eyeFacing = res[i].mesh[35][2] - res[i].mesh[263][2];
if (Math.abs(eyeFacing) < 10) gestures.push('facing camera'); if (Math.abs(eyeFacing) < 10) gestures.push({ face: i, gesture: 'facing camera' });
else gestures.push(`facing ${eyeFacing < 0 ? 'right' : 'left'}`); else gestures.push({ face: i, gesture: `facing ${eyeFacing < 0 ? 'right' : 'left'}` });
const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord const openLeft = Math.abs(res[i].mesh[374][1] - res[i].mesh[386][1]) / Math.abs(res[i].mesh[443][1] - res[i].mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openLeft < 0.2) gestures.push('blink left eye'); if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openRight < 0.2) gestures.push('blink right eye'); if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' });
const mouthOpen = Math.min(100, 500 * Math.abs(face.mesh[13][1] - face.mesh[14][1]) / Math.abs(face.mesh[10][1] - face.mesh[152][1])); const mouthOpen = Math.min(100, 500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1]) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]));
if (mouthOpen > 10) gestures.push(`mouth ${Math.trunc(mouthOpen)}% open`); if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` });
const chinDepth = face.mesh[152][2]; const chinDepth = res[i].mesh[152][2];
if (Math.abs(chinDepth) > 10) gestures.push(`head ${chinDepth < 0 ? 'up' : 'down'}`); if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` });
} }
} }
return gestures; return gestures;
@ -42,15 +42,15 @@ exports.face = (res) => {
exports.hand = (res) => { exports.hand = (res) => {
if (!res) return []; if (!res) return [];
const gestures = []; const gestures = [];
for (const hand of res) { for (const i in res) {
const fingers = []; const fingers = [];
for (const [finger, pos] of Object.entries(hand['annotations'])) { for (const [finger, pos] of Object.entries(res[i]['annotations'])) {
if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger
} }
if (fingers && fingers.length > 0) { if (fingers && fingers.length > 0) {
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a)); const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push(`${closest.name} forward ${highest.name} up`); gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` });
} }
} }
return gestures; return gestures;

2
src/human.js
View File

@ -408,7 +408,7 @@ class Human {
let gestureRes = []; let gestureRes = [];
if (this.config.gesture.enabled) { if (this.config.gesture.enabled) {
timeStamp = now(); timeStamp = now();
gestureRes = { face: gesture.face(faceRes), body: gesture.body(poseRes), hand: gesture.hand(handRes) }; gestureRes = [...gesture.face(faceRes), ...gesture.body(poseRes), ...gesture.hand(handRes)];
if (!this.config.async) this.perf.gesture = Math.trunc(now() - timeStamp); if (!this.config.async) this.perf.gesture = Math.trunc(now() - timeStamp);
else if (this.perf.gesture) delete this.perf.gesture; else if (this.perf.gesture) delete this.perf.gesture;
} }

2
wiki

@ -1 +1 @@
Subproject commit 333457d935d89f2559342b31907d04d091520604 Subproject commit bcac4981f7df29e367259caf6b3b73e5ecde6519