From 9bcfe23395f9c443009ddd95bb8b8aa7e07c1573 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 28 May 2021 15:53:51 -0400 Subject: [PATCH] added experimental face.rotation.gaze --- CHANGELOG.md | 5 +++- demo/index.js | 4 ++-- src/draw/draw.ts | 21 +++++++++++++++++ src/face.ts | 52 +++++++++++++++++++++++++++++++++++------- src/gesture/gesture.ts | 6 ++--- src/result.ts | 2 ++ 6 files changed, 75 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beb0f257..5c452286 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,10 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/05/27 mandic00@live.com +### **HEAD -> main** 2021/05/28 mandic00@live.com + + +### **origin/main** 2021/05/27 mandic00@live.com ### **1.9.4** 2021/05/27 mandic00@live.com diff --git a/demo/index.js b/demo/index.js index cb1f25e3..9f5db9b4 100644 --- a/demo/index.js +++ b/demo/index.js @@ -17,8 +17,8 @@ let human; const userConfig = { warmup: 'none', backend: 'webgl', - async: false, - cacheSensitivity: 0, + // async: false, + // cacheSensitivity: 0, filter: { enabled: false, flip: false, diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 747963fe..0561beb5 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -238,6 +238,27 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, dra ctx.fill(); } } + if (f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) { + const leftGaze = [ + f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]), + f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]), + ]; + ctx.beginPath(); + ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]); + ctx.strokeStyle = 'pink'; + ctx.lineTo(leftGaze[0], leftGaze[1]); + ctx.stroke(); + + const rightGaze = [ + f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]), + f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]), + ]; + ctx.beginPath(); + ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]); + ctx.strokeStyle = 'pink'; + ctx.lineTo(rightGaze[0], rightGaze[1]); + ctx.stroke(); + } } } } diff --git a/src/face.ts b/src/face.ts index bda10a32..c544dd9e 100644 --- a/src/face.ts +++ b/src/face.ts @@ -9,9 +9,40 @@ import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; import { Face } from './result'; -const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number] } => { - // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars - const degrees = (theta) => (theta * 180) / Math.PI; +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +const rad2deg = (theta) => (theta * 180) / Math.PI; + +const calculateGaze = (mesh): { angle: number, strength: number } => { + const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points + + const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes + const eyeRatio = 5; // factor to normalize changes x vs y + + const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis + const irisCenter = left ? mesh[473] : mesh[468]; + const eyeCenter = left // eye center is average of extreme points on x axis for both x and y, ignoring y extreme points as eyelids naturally open/close more when gazing up/down so relative point is less precise + ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] + : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; + const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions + ? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] + : [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; + + const eyeDiff = [ // x distance between extreme point and center point normalized with eye size + (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], + eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1], + ]; + const vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences + const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments + + // vectorAngle right=0*pi, up=1*pi/2, left=1*pi, down=3*pi/2 + return { angle: vectorAngle, strength: vectorLength }; +}; + +const calculateFaceAngle = (face, imageSize): { + angle: { pitch: number, yaw: number, roll: number }, + matrix: [number, number, number, number, number, number, number, number, number], + gaze: { angle: number, strength: number }, +} => { // const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360); const normalize = (v) => { // normalize vector const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); @@ -71,15 +102,16 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu return angle; }; + // initialize gaze and mesh const mesh = face.meshRaw; - if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] }; + if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { angle: 0, strength: 0 } }; - const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5; + const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5; // top, bottom, left, right const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ // make the xyz coordinates proportional, independent of the image/box size - pt[0] * image_size[0] / size, - pt[1] * image_size[1] / size, + pt[0] * imageSize[0] / size, + pt[1] * imageSize[1] / size, pt[2], ]); @@ -98,7 +130,11 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu ]; const angle = rotationMatrixToEulerAngle(matrix); // const angle = meshToEulerAngle(mesh); - return { angle, matrix }; + + // we have iris keypoints so we can calculate gaze direction + const gaze = mesh.length === 478 ? calculateGaze(mesh) : { angle: 0, strength: 0 }; + + return { angle, matrix, gaze }; }; export const detectFace = async (parent, input): Promise => { diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index aca4a696..e49e39d3 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -73,14 +73,12 @@ export const iris = (res): Gesture[] => { const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].box[3]; const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].box[3]; - if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01 || leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) center = false; + if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01 || leftIrisCenterY > 0.022 || rightIrisCenterY > 0.022) center = false; if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01) gestures.push({ iris: i, gesture: 'looking down' }); - if (leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) gestures.push({ iris: i, gesture: 'looking up' }); + if (leftIrisCenterY > 0.022 || rightIrisCenterY > 0.022) gestures.push({ iris: i, gesture: 'looking up' }); // still center; if (center) gestures.push({ iris: i, gesture: 'looking center' }); - - console.log(leftIrisCenterX, rightIrisCenterX, leftIrisCenterY, rightIrisCenterY, gestures); } return gestures; }; diff --git a/src/result.ts b/src/result.ts index 50fb97c0..22bb19bd 100644 --- a/src/result.ts +++ b/src/result.ts @@ -28,6 +28,7 @@ import { Tensor } from '../dist/tfjs.esm.js'; * - rotation: face rotiation that contains both angles and matrix used for 3d transformations * - angle: face angle as object with values for roll, yaw and pitch angles * - matrix: 3d transofrmation matrix as array of numeric values + * - gaze: gaze direction as object with values for agngle in radians and strengthss * - tensor: face tensor as Tensor object which contains detected face */ export interface Face { @@ -49,6 +50,7 @@ export interface Face { rotation: { angle: { roll: number, yaw: number, pitch: number }, matrix: [number, number, number, number, number, number, number, number, number], + gaze: { angle: number, strength: number }, } tensor: typeof Tensor, }