new face rotation calculations

CHANGELOG.md

@@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/03/27 mandic00@live.com
+### **HEAD -> main** 2021/03/28 animethemegadget@gmail.com
-
-
-### **origin/main** 2021/03/27 mandic00@live.com
 
+- rotationmatrixtoeulerangle, and fixes
+- experimental: add efficientpose
+- implement nanodet
+- face rotation matrix
 - start working on efficientpose
 
 ### **1.2.5** 2021/03/25 mandic00@live.com

src/draw/draw.ts

@@ -136,7 +136,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>) {
       const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
       labels.push(emotion.join(' '));
     }
-    if (f.angle && f.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.angle.roll) / 100} yaw:${Math.trunc(100 * f.angle.yaw) / 100} pitch:${Math.trunc(100 * f.angle.pitch) / 100}`);
+    if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
     if (labels.length === 0) labels.push('face');
     ctx.fillStyle = drawOptions.color;
     for (let i = labels.length - 1; i >= 0; i--) {

src/faceall.ts

@@ -8,76 +8,77 @@ import * as faceres from './faceres/faceres';
 
 type Tensor = typeof tf.Tensor;
 
-const calculateFaceAngle = (face, image_size): { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] } => {
+const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number] } => {
-  // normalize vector
+  // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
-  function normalize(v) {
+  const degrees = (theta) => (theta * 180) / Math.PI;
+  // const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360);
+  const normalize = (v) => { // normalize vector
     const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
     v[0] /= length;
     v[1] /= length;
     v[2] /= length;
     return v;
-  }
+  };
-
+  const subVectors = (a, b) => { // vector subtraction (a - b)
-  // vector subtraction (a - b)
-  function subVectors(a, b) {
     const x = a[0] - b[0];
     const y = a[1] - b[1];
     const z = a[2] - b[2];
     return [x, y, z];
-  }
+  };
-
+  const crossVectors = (a, b) => { // vector cross product (a x b)
-  // vector cross product (a x b)
-  function crossVectors(a, b) {
     const x = a[1] * b[2] - a[2] * b[1];
     const y = a[2] * b[0] - a[0] * b[2];
     const z = a[0] * b[1] - a[1] * b[0];
     return [x, y, z];
-  }
+  };
-
+  // 3x3 rotation matrix to Euler angles based on https://www.geometrictools.com/Documentation/EulerAngles.pdf
-  // 3x3 rotation matrix to Euler angles
+  const rotationMatrixToEulerAngle = (r) => {
-  // https://www.geometrictools.com/Documentation/EulerAngles.pdf
+    // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
-  function rotationMatrixToEulerAngle(r) {
+    const [r00, r01, r02, r10, r11, r12, r20, r21, r22] = r;
-    // r01 is not used yet (no-unused-vars)
-    // eslint-disable-next-line
-    const [ r00, r01, r02, r10, r11, r12, r20, r21, r22 ] = r;
-    const pi = Math.PI;
-
     let thetaX; let thetaY; let thetaZ;
-
+    if (r10 < 1) { // YZX calculation
-    // YZX
-    if (r10 < 1) {
       if (r10 > -1) {
         thetaZ = Math.asin(r10);
-        // thetaY = Math.atan2(-r20, r00);
         thetaX = Math.atan2(-r12, r11);
       } else {
-        thetaZ = -pi / 2;
+        thetaZ = -Math.PI / 2;
-        // thetaY = -Math.atan2(r21, r22);
         thetaX = 0;
       }
     } else {
-      thetaZ = pi / 2;
+      thetaZ = Math.PI / 2;
-      // thetaY = Math.atan2(r21, r22);
       thetaX = 0;
     }
-
+    if (r02 < 1) { // compensate Y rotation which is not accurate and too small in YZX calculation
-    // compensate Y rotation (from XYZ rotation order routine) which is not accurate and too small in YZX calculation
-    if (r02 < 1) {
       if (r02 > -1) {
         thetaY = Math.asin(r02);
       } else {
-        thetaY = -pi / 2;
+        thetaY = -Math.PI / 2;
       }
     } else {
-      thetaY = pi / 2;
+      thetaY = Math.PI / 2;
     }
-
+    return { pitch: 2 * -thetaX, yaw: 2 * -thetaY, roll: 2 * -thetaZ };
-    // pitch, yaw, row
+  };
-    return [-thetaX, -thetaY, -thetaZ];
+  // simple Euler angle calculation based existing 3D mesh
-  }
+  // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+  const meshToEulerAngle = (mesh) => {
+    const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1);
+    // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+    const angle = {
+      // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees
+      // value of 0 means center
+      // pitch is face move up/down
+      pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face
+      // yaw is face turn left/right
+      yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye
+      // roll is face lean left/right
+      roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye
+    };
+    return angle;
+  };
 
   const mesh = face.meshRaw;
-  if (!mesh || mesh.length < 300) return { pitch: 0, yaw: 0, row: 0, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] };
+  if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] };
 
   const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5;
   // top, bottom, left, right
@@ -96,20 +97,14 @@ const calculateFaceAngle = (face, image_size): { pitch: number, yaw: number, row
 
   // Rotation Matrix from Axis Vectors - http://renderdan.blogspot.com/2006/05/rotation-matrix-from-axis-vectors.html
   // 3x3 rotation matrix is flatten to array in row-major order. Note that the rotation represented by this matrix is inverted.
-  const r_matrix = [
+  const matrix: [number, number, number, number, number, number, number, number, number] = [
     x_axis[0], x_axis[1], x_axis[2],
     y_axis[0], y_axis[1], y_axis[2],
     z_axis[0], z_axis[1], z_axis[2],
   ];
-
+  const angle = rotationMatrixToEulerAngle(matrix);
-  const [pitch, yaw, row] = rotationMatrixToEulerAngle(r_matrix);
+  // const angle = meshToEulerAngle(mesh);
-
+  return { angle, matrix };
-  return {
-    pitch,
-    yaw,
-    row,
-    matrix: r_matrix,
-  };
 };
 
 export const detectFace = async (parent, input): Promise<any> => {
@@ -136,7 +131,10 @@ export const detectFace = async (parent, input): Promise<any> => {
       emotion: string,
       embedding: number[],
       iris: number,
-      angle: { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] },
+      rotation: {
+        angle: { pitch: number, yaw: number, roll: number },
+        matrix: [number, number, number, number, number, number, number, number, number]
+      },
       tensor: Tensor,
     }> = [];
   parent.state = 'run:face';
@@ -153,7 +151,7 @@ export const detectFace = async (parent, input): Promise<any> => {
       continue;
     }
 
-    const angle = calculateFaceAngle(face, [input.shape[2], input.shape[1]]);
+    const rotation = calculateFaceAngle(face, [input.shape[2], input.shape[1]]);
 
     // run age, inherits face from blazeface
     parent.analyze('Start Age:');
@@ -240,7 +238,7 @@ export const detectFace = async (parent, input): Promise<any> => {
       embedding: descRes.descriptor || embeddingRes,
       emotion: emotionRes,
       iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0,
-      angle,
+      rotation,
       tensor: parent.config.face.detector.return ? face.image?.squeeze() : null,
     });
     // dispose original face tensor

wiki

@@ -1 +1 @@
-Subproject commit 11e68676b2bf9aadd7ff1e2cc80dd4a70052f9d3
+Subproject commit aae30ba3904eb7d76847895e11c55926e35ad00b