From a5b78acdefedca80aafdf01d9db63a6f5774d22c Mon Sep 17 00:00:00 2001 From: ButzYung Date: Sun, 28 Mar 2021 18:19:25 +0800 Subject: [PATCH] rotationMatrixToEulerAngle, and fixes --- src/blazeface/facemesh.ts | 6 +-- src/faceall.ts | 103 ++++++++++++++++++++++++++++++-------- 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/src/blazeface/facemesh.ts b/src/blazeface/facemesh.ts index ccf0ab82..b042f755 100644 --- a/src/blazeface/facemesh.ts +++ b/src/blazeface/facemesh.ts @@ -19,11 +19,9 @@ export class MediaPipeFaceMesh { for (const prediction of (predictions || [])) { if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing const mesh = prediction.coords ? prediction.coords.arraySync() : []; - // this should be the best way to get the meshRaw with x and y fitting the box with aspect ratio kept (values still normalized to 0..1) - const size = prediction.box ? Math.max((prediction.box.endPoint[0] - prediction.box.startPoint[0]), (prediction.box.endPoint[1] - prediction.box.startPoint[1])) / 1.5 : 1; const meshRaw = mesh.map((pt) => [ - pt[0] / size, - pt[1] / size, + pt[0] / input.shape[2], + pt[1] / input.shape[1], pt[2] / this.facePipeline.meshSize, ]); const annotations = {}; diff --git a/src/faceall.ts b/src/faceall.ts index 2a25fa0c..0007c97b 100644 --- a/src/faceall.ts +++ b/src/faceall.ts @@ -8,45 +8,108 @@ import * as faceres from './faceres/faceres'; type Tensor = typeof tf.Tensor; -const calculateFaceAngle = (mesh): { matrix: [number, number, number, number, number, number, number, number, number, number, number, number, number, number, number, number] } => { - if (!mesh || mesh.length < 300) return { matrix: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1] }; - - const normalize = (v) => { +const calculateFaceAngle = (face, image_size): { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] } => { + // normalize vector + function normalize(v) { const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); v[0] /= length; v[1] /= length; v[2] /= length; return v; - }; + } - const subVectors = (a, b) => { + // vector subtraction (a - b) + function subVectors(a, b) { const x = a[0] - b[0]; const y = a[1] - b[1]; const z = a[2] - b[2]; return [x, y, z]; - }; + } - const crossVectors = (a, b) => { + // vector cross product (a x b) + function crossVectors(a, b) { const x = a[1] * b[2] - a[2] * b[1]; const y = a[2] * b[0] - a[0] * b[2]; const z = a[0] * b[1] - a[1] * b[0]; return [x, y, z]; - }; + } - const y_axis = normalize(subVectors(mesh[152], mesh[10])); - let x_axis = normalize(subVectors(mesh[454], mesh[234])); + // 3x3 rotation matrix to Euler angles + // https://www.geometrictools.com/Documentation/EulerAngles.pdf + function rotationMatrixToEulerAngle(r) { + // r01 is not used yet (no-unused-vars) + // eslint-disable-next-line + const [ r00, r01, r02, r10, r11, r12, r20, r21, r22 ] = r; + const pi = Math.PI; + + let thetaX; let thetaY; let thetaZ; + + // YZX + if (r10 < 1) { + if (r10 > -1) { + thetaZ = Math.asin(r10); + thetaY = Math.atan2(-r20, r00); + thetaX = Math.atan2(-r12, r11); + } else { + thetaZ = -pi / 2; + thetaY = -Math.atan2(r21, r22); + thetaX = 0; + } + } else { + thetaZ = pi / 2; + thetaY = Math.atan2(r21, r22); + thetaX = 0; + } + + // compensate Y rotation (from XYZ rotation order routine) which is not accurate and too small in YZX calculation + if (r02 < 1) { + if (r02 > -1) { + thetaY = Math.asin(r02); + } else { + thetaY = -pi / 2; + } + } else { + thetaY = pi / 2; + } + + // pitch, yaw, row + return [-thetaX, -thetaY, -thetaZ]; + } + + const mesh = face.meshRaw; + if (!mesh || mesh.length < 300) return { pitch: 0, yaw: 0, row: 0, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] }; + + const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5; + // top, bottom, left, right + const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ + // make the xyz coordinates proportional, independent of the image/box size + pt[0] * image_size[0] / size, + pt[1] * image_size[1] / size, + pt[2], + ]); + + const y_axis = normalize(subVectors(pts[1], pts[0])); + let x_axis = normalize(subVectors(pts[3], pts[2])); const z_axis = normalize(crossVectors(x_axis, y_axis)); // adjust x_axis to make sure that all axes are perpendicular to each other x_axis = crossVectors(y_axis, z_axis); // Rotation Matrix from Axis Vectors - http://renderdan.blogspot.com/2006/05/rotation-matrix-from-axis-vectors.html - // note that the rotation matrix is flatten to array in column-major order (instead of row-major order), which directly fits three.js Matrix4.fromArray function - return { matrix: [ - x_axis[0], y_axis[0], z_axis[0], 0, - x_axis[1], y_axis[1], z_axis[1], 0, - x_axis[2], y_axis[2], z_axis[2], 0, - 0, 0, 0, 1, - ] }; + // 3x3 rotation matrix is flatten to array in row-major order. Note that the rotation represented by this matrix is inverted. + const r_matrix = [ + x_axis[0], x_axis[1], x_axis[2], + y_axis[0], y_axis[1], y_axis[2], + z_axis[0], z_axis[1], z_axis[2], + ]; + + const [pitch, yaw, row] = rotationMatrixToEulerAngle(r_matrix); + + return { + pitch, + yaw, + row, + matrix: r_matrix, + }; }; export const detectFace = async (parent, input): Promise => { @@ -73,7 +136,7 @@ export const detectFace = async (parent, input): Promise => { emotion: string, embedding: number[], iris: number, - angle: { matrix:[number, number, number, number, number, number, number, number, number, number, number, number, number, number, number, number] }, + angle: { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] }, tensor: Tensor, }> = []; parent.state = 'run:face'; @@ -90,7 +153,7 @@ export const detectFace = async (parent, input): Promise => { continue; } - const angle = calculateFaceAngle(face.meshRaw); + const angle = calculateFaceAngle(face, [input.shape[2], input.shape[1]]); // run age, inherits face from blazeface parent.analyze('Start Age:');