ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added experimental face.rotation.gaze

pull/280/head
Vladimir Mandic 2021-05-28 15:53:51 -04:00
parent 54ddb2e2be
commit 1678d3192f
6 changed files with 75 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -9,7 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/05/27 mandic00@live.com
### **HEAD -> main** 2021/05/28 mandic00@live.com
### **origin/main** 2021/05/27 mandic00@live.com
### **1.9.4** 2021/05/27 mandic00@live.com

4
demo/index.js
View File

@ -17,8 +17,8 @@ let human;
const userConfig = {
warmup: 'none',
backend: 'webgl',
async: false,
cacheSensitivity: 0,
// async: false,
// cacheSensitivity: 0,
filter: {
enabled: false,
flip: false,

21
src/draw/draw.ts
View File

@ -238,6 +238,27 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.fill();
}
}
if (f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) {
const leftGaze = [
f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]),
f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]),
];
ctx.beginPath();
ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]);
ctx.strokeStyle = 'pink';
ctx.lineTo(leftGaze[0], leftGaze[1]);
ctx.stroke();
const rightGaze = [
f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]),
f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]),
];
ctx.beginPath();
ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]);
ctx.strokeStyle = 'pink';
ctx.lineTo(rightGaze[0], rightGaze[1]);
ctx.stroke();
}
}
}
}

52
src/face.ts
View File

@ -9,9 +9,40 @@ import * as emotion from './emotion/emotion';
import * as faceres from './faceres/faceres';
import { Face } from './result';
const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number] } => {
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const degrees = (theta) => (theta * 180) / Math.PI;
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const rad2deg = (theta) => (theta * 180) / Math.PI;
const calculateGaze = (mesh): { angle: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes
const eyeRatio = 5; // factor to normalize changes x vs y
const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
const irisCenter = left ? mesh[473] : mesh[468];
const eyeCenter = left // eye center is average of extreme points on x axis for both x and y, ignoring y extreme points as eyelids naturally open/close more when gazing up/down so relative point is less precise
? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2]
: [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2];
const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions
? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]]
: [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]];
const eyeDiff = [ // x distance between extreme point and center point normalized with eye size
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1],
];
const vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments
// vectorAngle right=0*pi, up=1*pi/2, left=1*pi, down=3*pi/2
return { angle: vectorAngle, strength: vectorLength };
};
const calculateFaceAngle = (face, imageSize): {
angle: { pitch: number, yaw: number, roll: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number },
} => {
// const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360);
const normalize = (v) => { // normalize vector
const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
@ -71,15 +102,16 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu
return angle;
};
// initialize gaze and mesh
const mesh = face.meshRaw;
if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] };
if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { angle: 0, strength: 0 } };
const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5;
const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5;
// top, bottom, left, right
const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
// make the xyz coordinates proportional, independent of the image/box size
pt[0] * image_size[0] / size,
pt[1] * image_size[1] / size,
pt[0] * imageSize[0] / size,
pt[1] * imageSize[1] / size,
pt[2],
]);
@ -98,7 +130,11 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu
];
const angle = rotationMatrixToEulerAngle(matrix);
// const angle = meshToEulerAngle(mesh);
return { angle, matrix };
// we have iris keypoints so we can calculate gaze direction
const gaze = mesh.length === 478 ? calculateGaze(mesh) : { angle: 0, strength: 0 };
return { angle, matrix, gaze };
};
export const detectFace = async (parent, input): Promise<Face[]> => {

6
src/gesture/gesture.ts
View File

@ -73,14 +73,12 @@ export const iris = (res): Gesture[] => {
const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].box[3];
const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].box[3];
if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01 || leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) center = false;
if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01 || leftIrisCenterY > 0.022 || rightIrisCenterY > 0.022) center = false;
if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01) gestures.push({ iris: i, gesture: 'looking down' });
if (leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) gestures.push({ iris: i, gesture: 'looking up' });
if (leftIrisCenterY > 0.022 || rightIrisCenterY > 0.022) gestures.push({ iris: i, gesture: 'looking up' });
// still center;
if (center) gestures.push({ iris: i, gesture: 'looking center' });
console.log(leftIrisCenterX, rightIrisCenterX, leftIrisCenterY, rightIrisCenterY, gestures);
}
return gestures;
};

2
src/result.ts
View File

@ -28,6 +28,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations
* - angle: face angle as object with values for roll, yaw and pitch angles
* - matrix: 3d transofrmation matrix as array of numeric values
* - gaze: gaze direction as object with values for agngle in radians and strengthss
* - tensor: face tensor as Tensor object which contains detected face
*/
export interface Face {
@ -49,6 +50,7 @@ export interface Face {
rotation: {
angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number },
}
tensor: typeof Tensor,
}