rotationMatrixToEulerAngle, and fixes

src/blazeface/facemesh.ts

@@ -19,11 +19,9 @@ export class MediaPipeFaceMesh {
     for (const prediction of (predictions || [])) {
       if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
       const mesh = prediction.coords ? prediction.coords.arraySync() : [];
-      // this should be the best way to get the meshRaw with x and y fitting the box with aspect ratio kept (values still normalized to 0..1)
-      const size = prediction.box ? Math.max((prediction.box.endPoint[0] - prediction.box.startPoint[0]), (prediction.box.endPoint[1] - prediction.box.startPoint[1])) / 1.5 : 1;
       const meshRaw = mesh.map((pt) => [
-        pt[0] / size,
-        pt[1] / size,
+        pt[0] / input.shape[2],
+        pt[1] / input.shape[1],
         pt[2] / this.facePipeline.meshSize,
       ]);
       const annotations = {};

src/faceall.ts

@@ -8,45 +8,108 @@ import * as faceres from './faceres/faceres';
 
 type Tensor = typeof tf.Tensor;
 
-const calculateFaceAngle = (mesh): { matrix: [number, number, number, number, number, number, number, number, number, number, number, number, number, number, number, number] } => {
-  if (!mesh || mesh.length < 300) return { matrix: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] };
-
-  const normalize = (v) => {
+const calculateFaceAngle = (face, image_size): { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] } => {
+  // normalize vector
+  function normalize(v) {
     const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
     v[0] /= length;
     v[1] /= length;
     v[2] /= length;
     return v;
-  };
+  }
 
-  const subVectors = (a, b) => {
+  // vector subtraction (a - b)
+  function subVectors(a, b) {
     const x = a[0] - b[0];
     const y = a[1] - b[1];
     const z = a[2] - b[2];
     return [x, y, z];
-  };
+  }
 
-  const crossVectors = (a, b) => {
+  // vector cross product (a x b)
+  function crossVectors(a, b) {
     const x = a[1] * b[2] - a[2] * b[1];
     const y = a[2] * b[0] - a[0] * b[2];
     const z = a[0] * b[1] - a[1] * b[0];
     return [x, y, z];
-  };
+  }
 
-  const y_axis = normalize(subVectors(mesh[152], mesh[10]));
-  let x_axis = normalize(subVectors(mesh[454], mesh[234]));
+  // 3x3 rotation matrix to Euler angles
+  // https://www.geometrictools.com/Documentation/EulerAngles.pdf
+  function rotationMatrixToEulerAngle(r) {
+    // r01 is not used yet (no-unused-vars)
+    // eslint-disable-next-line
+    const [ r00, r01, r02, r10, r11, r12, r20, r21, r22 ] = r;
+    const pi = Math.PI;
+
+    let thetaX; let thetaY; let thetaZ;
+
+    // YZX
+    if (r10 < 1) {
+      if (r10 > -1) {
+        thetaZ = Math.asin(r10);
+        thetaY = Math.atan2(-r20, r00);
+        thetaX = Math.atan2(-r12, r11);
+      } else {
+        thetaZ = -pi / 2;
+        thetaY = -Math.atan2(r21, r22);
+        thetaX = 0;
+      }
+    } else {
+      thetaZ = pi / 2;
+      thetaY = Math.atan2(r21, r22);
+      thetaX = 0;
+    }
+
+    // compensate Y rotation (from XYZ rotation order routine) which is not accurate and too small in YZX calculation
+    if (r02 < 1) {
+      if (r02 > -1) {
+        thetaY = Math.asin(r02);
+      } else {
+        thetaY = -pi / 2;
+      }
+    } else {
+      thetaY = pi / 2;
+    }
+
+    // pitch, yaw, row
+    return [-thetaX, -thetaY, -thetaZ];
+  }
+
+  const mesh = face.meshRaw;
+  if (!mesh || mesh.length < 300) return { pitch: 0, yaw: 0, row: 0, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] };
+
+  const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5;
+  // top, bottom, left, right
+  const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
+    // make the xyz coordinates proportional, independent of the image/box size
+    pt[0] * image_size[0] / size,
+    pt[1] * image_size[1] / size,
+    pt[2],
+  ]);
+
+  const y_axis = normalize(subVectors(pts[1], pts[0]));
+  let x_axis = normalize(subVectors(pts[3], pts[2]));
   const z_axis = normalize(crossVectors(x_axis, y_axis));
   // adjust x_axis to make sure that all axes are perpendicular to each other
   x_axis = crossVectors(y_axis, z_axis);
 
   // Rotation Matrix from Axis Vectors - http://renderdan.blogspot.com/2006/05/rotation-matrix-from-axis-vectors.html
-  // note that the rotation matrix is flatten to array in column-major order (instead of row-major order), which directly fits three.js Matrix4.fromArray function
-  return { matrix: [
-    x_axis[0], y_axis[0], z_axis[0], 0,
-    x_axis[1], y_axis[1], z_axis[1], 0,
-    x_axis[2], y_axis[2], z_axis[2], 0,
-    0, 0, 0, 1,
-  ] };
+  // 3x3 rotation matrix is flatten to array in row-major order. Note that the rotation represented by this matrix is inverted.
+  const r_matrix = [
+    x_axis[0], x_axis[1], x_axis[2],
+    y_axis[0], y_axis[1], y_axis[2],
+    z_axis[0], z_axis[1], z_axis[2],
+  ];
+
+  const [pitch, yaw, row] = rotationMatrixToEulerAngle(r_matrix);
+
+  return {
+    pitch,
+    yaw,
+    row,
+    matrix: r_matrix,
+  };
 };
 
 export const detectFace = async (parent, input): Promise<any> => {
@@ -73,7 +136,7 @@ export const detectFace = async (parent, input): Promise<any> => {
       emotion: string,
       embedding: number[],
       iris: number,
-      angle: { matrix:[number, number, number, number, number, number, number, number, number, number, number, number, number, number, number, number] },
+      angle: { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] },
       tensor: Tensor,
     }> = [];
   parent.state = 'run:face';
@@ -90,7 +153,7 @@ export const detectFace = async (parent, input): Promise<any> => {
       continue;
     }
 
-    const angle = calculateFaceAngle(face.meshRaw);
+    const angle = calculateFaceAngle(face, [input.shape[2], input.shape[1]]);
 
     // run age, inherits face from blazeface
     parent.analyze('Start Age:');