Fix face angles (yaw, pitch, & roll) accuracy

Previouly derived aforementioned angles correctly seemed inaccurate and somewhat unusable (given their output was in radians). This update uses the a person's mesh positions, and chooses specific points for accurate results. It also adds directionality of the movements (_e.g. pitching head backwards is a negative result, as is rolling head to the left).

The webcam.js file has also been updated to showcase the correct output in degrees (reducing potential user confusion)

Comitter: Sohaib Ahmed <sohaibi.ahmed@icloud.com>

demo/webcam.js

@@ -53,12 +53,12 @@ function drawFaces(canvas, data, fps) {
     ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 59);
     ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 41);
     ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 23);
-    ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 5);
+    ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 5);
     ctx.fillStyle = 'lightblue';
     ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 60);
     ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 42);
     ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 24);
-    ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 6);
+    ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 6);
     // draw face points for each face
     ctx.globalAlpha = 0.8;
     ctx.fillStyle = 'lightblue';

src/factories/WithFaceLandmarks.ts

@@ -5,64 +5,130 @@ import { isWithFaceDetection, WithFaceDetection } from './WithFaceDetection';
 
 export type WithFaceLandmarks<
   TSource extends WithFaceDetection<{}>,
-  TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 > = TSource & {
+  TFaceLandmarks extends FaceLandmarks = FaceLandmarks68
-    landmarks: TFaceLandmarks,
+> = TSource & {
-    unshiftedLandmarks: TFaceLandmarks,
+  landmarks: TFaceLandmarks;
-    alignedRect: FaceDetection,
+  unshiftedLandmarks: TFaceLandmarks;
-    angle: { roll: number | undefined, pitch: number | undefined, yaw: number | undefined },
+  alignedRect: FaceDetection;
-  }
+  angle: {
+    roll: number | undefined;
+    pitch: number | undefined;
+    yaw: number | undefined;
+  };
+};
 
-export function isWithFaceLandmarks(obj: any): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
+export function isWithFaceLandmarks(
-  return isWithFaceDetection(obj)
+  obj: any,
+): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
+  return (
+    isWithFaceDetection(obj)
     // eslint-disable-next-line dot-notation
     && obj['landmarks'] instanceof FaceLandmarks
     // eslint-disable-next-line dot-notation
     && obj['unshiftedLandmarks'] instanceof FaceLandmarks
     // eslint-disable-next-line dot-notation
-    && obj['alignedRect'] instanceof FaceDetection;
+    && obj['alignedRect'] instanceof FaceDetection
+  );
 }
 
 function calculateFaceAngle(mesh) {
-  // returns the angle in the plane (in radians) between the positive x-axis and the ray from (0,0) to the point (x,y)
+  /*
-  const radians = (a1, a2, b1, b2) => (Math.atan2(b2 - a2, b1 - a1) % Math.PI);
+    AUTHORED BY: SOHAIB AHMED
-  // convert radians to degrees
+    https://github.com/TheSohaibAhmed/
-  // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+  */
-  const degrees = (theta) => (theta * 180) / Math.PI;
 
-  const angle = { roll: <number | undefined>undefined, pitch: <number | undefined>undefined, yaw: <number | undefined>undefined };
+  // Helper to convert radians to degrees
+  // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+  const degrees = (radians) => (radians * 180) / Math.PI;
+
+  const angle = {
+    roll: <number | undefined>undefined,
+    pitch: <number | undefined>undefined,
+    yaw: <number | undefined>undefined,
+  };
 
   if (!mesh || !mesh._positions || mesh._positions.length !== 68) return angle;
   const pt = mesh._positions;
 
-  // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees
+  function calcLengthBetweenTwoPoints(a, b) {
-  // value of 0 means center
+    return Math.sqrt((a._x - b._x) ** 2 + (a._y - b._y) ** 2);
+  }
+  const calcYaw = (leftPoint, midPoint, rightPoint) => {
+    // Calc x-distance from left side of the face ("ear") to facial midpoint ("nose")
+    const leftToMidpoint = Math.floor(leftPoint._x - midPoint._x);
 
-  // roll is face lean from left to right
+    // Calc x-distance from facial midpoint ("nose") to the right side of the face ("ear")
-  // comparing x,y of outside corners of leftEye and rightEye
+    const rightToMidpoint = Math.floor(midPoint._x - rightPoint._x);
-  angle.roll = -radians(pt[36]._x, pt[36]._y, pt[45]._x, pt[45]._y);
 
-  // pitch is face turn from left right
+    // Difference in distances coincidentally approximates to angles
-  // comparing x distance of top of nose to left and right edge of face
+    const distanceApproximatesToAngle = leftToMidpoint - rightToMidpoint;
-  // precision is lacking since coordinates are not precise enough
+    return distanceApproximatesToAngle;
-  angle.pitch = radians(0, Math.abs(pt[0]._x - pt[30]._x) / pt[30]._x, Math.PI, Math.abs(pt[16]._x - pt[30]._x) / pt[30]._x);
+  };
 
-  // yaw is face move from up to down
+  const calcRoll = (lever, pivot) => {
-  // comparing size of the box around the face with top and bottom of detected landmarks
+    // When rolling, the head seems to pivot from the nose/lips/chin area.
-  // silly hack, but this gives us face compression on y-axis
+    // So, we'll choose any two points from the facial midline, where the first point should be the pivot, and the other "lever"
-  // e.g., tilting head up hides the forehead that doesn't have any landmarks so ratio drops
+    // Plan/Execution: get the hypotenuse & opposite sides of a 90deg triangle ==> Calculate angle in radians
-  const bottom = pt.reduce((prev, cur) => (prev < cur._y ? prev : cur._y), +Infinity);
+
-  const top = pt.reduce((prev, cur) => (prev > cur._y ? prev : cur._y), -Infinity);
+    const hypotenuse = Math.hypot(pivot._x - lever._x, pivot._y - lever._y);
-  angle.yaw = Math.PI * (mesh._imgDims._height / (top - bottom) / 1.40 - 1);
+    const opposite = pivot._y - lever._y;
+    const angleInRadians = Math.asin(opposite / hypotenuse);
+    const angleInDegrees = degrees(angleInRadians);
+    const normalizeAngle = Math.floor(90 - angleInDegrees);
+    // If lever more to the left of the pivot, then we're tilting left
+    // "-" is negative direction. "+", or absence of a sign is positive direction
+    const tiltDirection = pivot._x - lever._x < 0 ? -1 : 1;
+    const result = normalizeAngle * tiltDirection;
+    return result;
+  };
+
+  const calcPitch = (leftPoint, midPoint, rightPoint) => {
+    // Theory: While pitching, the nose is the most salient point --> That's what we'll use to make a trianle.
+    // The "base" is between point that don't move when we pitch our head (i.e. an imaginary line running ear to ear through the nose).
+    // Executuin: Get the opposite & adjacent lengths of the triangle from the ear's perspective. Use it to get angle.
+
+    const base = calcLengthBetweenTwoPoints(leftPoint, rightPoint);
+    // adjecent is base/2 technically.
+    const baseCoords = {
+      _x: (leftPoint._x + rightPoint._x) / 2,
+      _y: (leftPoint._y + rightPoint._y) / 2,
+    };
+    const midToBaseLength = calcLengthBetweenTwoPoints(midPoint, baseCoords);
+    const angleInRadians = Math.atan(midToBaseLength / base);
+    const angleInDegrees = Math.floor(degrees(angleInRadians));
+    // Account for directionality.
+    // pitch forwards (_i.e. tilting your head forwards) is positive (or no sign); backward is negative.
+    const direction = baseCoords._y - midPoint._y < 0 ? -1 : 1;
+    const result = angleInDegrees * direction;
+    return result;
+  };
+
+  angle.roll = calcRoll(pt[27], pt[66]);
+  angle.pitch = calcPitch(pt[14], pt[30], pt[2]);
+  angle.yaw = calcYaw(pt[14], pt[33], pt[2]);
 
   return angle;
 }
 
-export function extendWithFaceLandmarks<TSource extends WithFaceDetection<{}>, TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 >(sourceObj: TSource, unshiftedLandmarks: TFaceLandmarks): WithFaceLandmarks<TSource, TFaceLandmarks> {
+export function extendWithFaceLandmarks<
+  TSource extends WithFaceDetection<{}>,
+  TFaceLandmarks extends FaceLandmarks = FaceLandmarks68
+>(
+  sourceObj: TSource,
+  unshiftedLandmarks: TFaceLandmarks,
+): WithFaceLandmarks<TSource, TFaceLandmarks> {
   const { box: shift } = sourceObj.detection;
-  const landmarks = unshiftedLandmarks.shiftBy<TFaceLandmarks>(shift.x, shift.y);
+  const landmarks = unshiftedLandmarks.shiftBy<TFaceLandmarks>(
+    shift.x,
+    shift.y,
+  );
   const rect = landmarks.align();
   const { imageDims } = sourceObj.detection;
-  const alignedRect = new FaceDetection(sourceObj.detection.score, rect.rescale(imageDims.reverse()), imageDims);
+  const alignedRect = new FaceDetection(
+    sourceObj.detection.score,
+    rect.rescale(imageDims.reverse()),
+    imageDims,
+  );
   const angle = calculateFaceAngle(unshiftedLandmarks);
 
   const extension = {