/** * Module that analyzes person age * Obsolete */ import { log, now } from './helpers'; import * as facemesh from './blazeface/facemesh'; import * as emotion from './emotion/emotion'; import * as faceres from './faceres/faceres'; import { Face } from './result'; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const rad2deg = (theta) => (theta * 180) / Math.PI; const calculateGaze = (mesh): { angle: number, strength: number } => { const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points const offsetIris = [0, 0]; // tbd: iris center may not align with average of eye extremes const eyeRatio = 5; // factor to normalize changes x vs y const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis const irisCenter = left ? mesh[473] : mesh[468]; const eyeCenter = left // eye center is average of extreme points on x axis for both x and y, ignoring y extreme points as eyelids naturally open/close more when gazing up/down so relative point is less precise ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions ? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] : [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; const eyeDiff = [ // x distance between extreme point and center point normalized with eye size (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1], ]; const vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments // vectorAngle right=0*pi, up=1*pi/2, left=1*pi, down=3*pi/2 return { angle: vectorAngle, strength: vectorLength }; }; const calculateFaceAngle = (face, imageSize): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number], gaze: { angle: number, strength: number }, } => { // const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360); const normalize = (v) => { // normalize vector const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); v[0] /= length; v[1] /= length; v[2] /= length; return v; }; const subVectors = (a, b) => { // vector subtraction (a - b) const x = a[0] - b[0]; const y = a[1] - b[1]; const z = a[2] - b[2]; return [x, y, z]; }; const crossVectors = (a, b) => { // vector cross product (a x b) const x = a[1] * b[2] - a[2] * b[1]; const y = a[2] * b[0] - a[0] * b[2]; const z = a[0] * b[1] - a[1] * b[0]; return [x, y, z]; }; // 3x3 rotation matrix to Euler angles based on https://www.geometrictools.com/Documentation/EulerAngles.pdf const rotationMatrixToEulerAngle = (r) => { // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const [r00, r01, r02, r10, r11, r12, r20, r21, r22] = r; let thetaX; let thetaY; let thetaZ; if (r10 < 1) { // YZX calculation if (r10 > -1) { thetaZ = Math.asin(r10); thetaY = Math.atan2(-r20, r00); thetaX = Math.atan2(-r12, r11); } else { thetaZ = -Math.PI / 2; thetaY = -Math.atan2(r21, r22); thetaX = 0; } } else { thetaZ = Math.PI / 2; thetaY = Math.atan2(r21, r22); thetaX = 0; } return { pitch: 2 * -thetaX, yaw: 2 * -thetaY, roll: 2 * -thetaZ }; }; // simple Euler angle calculation based existing 3D mesh // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const meshToEulerAngle = (mesh) => { const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1); // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars const angle = { // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees, value of 0 means center // pitch is face move up/down pitch: radians(mesh[10][1], mesh[10][2], mesh[152][1], mesh[152][2]), // looking at y,z of top and bottom points of the face // yaw is face turn left/right yaw: radians(mesh[33][0], mesh[33][2], mesh[263][0], mesh[263][2]), // looking at x,z of outside corners of leftEye and rightEye // roll is face lean left/right roll: radians(mesh[33][0], mesh[33][1], mesh[263][0], mesh[263][1]), // looking at x,y of outside corners of leftEye and rightEye }; return angle; }; // initialize gaze and mesh const mesh = face.meshRaw; if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { angle: 0, strength: 0 } }; const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5; // top, bottom, left, right const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [ // make the xyz coordinates proportional, independent of the image/box size pt[0] * imageSize[0] / size, pt[1] * imageSize[1] / size, pt[2], ]); const y_axis = normalize(subVectors(pts[1], pts[0])); let x_axis = normalize(subVectors(pts[3], pts[2])); const z_axis = normalize(crossVectors(x_axis, y_axis)); // adjust x_axis to make sure that all axes are perpendicular to each other x_axis = crossVectors(y_axis, z_axis); // Rotation Matrix from Axis Vectors - http://renderdan.blogspot.com/2006/05/rotation-matrix-from-axis-vectors.html // 3x3 rotation matrix is flatten to array in row-major order. Note that the rotation represented by this matrix is inverted. const matrix: [number, number, number, number, number, number, number, number, number] = [ x_axis[0], x_axis[1], x_axis[2], y_axis[0], y_axis[1], y_axis[2], z_axis[0], z_axis[1], z_axis[2], ]; const angle = rotationMatrixToEulerAngle(matrix); // const angle = meshToEulerAngle(mesh); // we have iris keypoints so we can calculate gaze direction const gaze = mesh.length === 478 ? calculateGaze(mesh) : { angle: 0, strength: 0 }; return { angle, matrix, gaze }; }; export const detectFace = async (parent, input): Promise => { // run facemesh, includes blazeface and iris // eslint-disable-next-line no-async-promise-executor let timeStamp; let ageRes; let genderRes; let emotionRes; let embeddingRes; let descRes; const faceRes: Array = []; parent.state = 'run:face'; timeStamp = now(); const faces = await facemesh.predict(input, parent.config); parent.perf.face = Math.trunc(now() - timeStamp); if (!faces) return []; // for (const face of faces) { for (let i = 0; i < faces.length; i++) { parent.analyze('Get Face'); // is something went wrong, skip the face if (!faces[i].image || faces[i].image.isDisposedInternal) { log('Face object is disposed:', faces[i].image); continue; } const rotation = calculateFaceAngle(faces[i], [input.shape[2], input.shape[1]]); // run emotion, inherits face from blazeface parent.analyze('Start Emotion:'); if (parent.config.async) { emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].image, parent.config, i, faces.length) : {}; } else { parent.state = 'run:emotion'; timeStamp = now(); emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].image, parent.config, i, faces.length) : {}; parent.perf.emotion = Math.trunc(now() - timeStamp); } parent.analyze('End Emotion:'); // run emotion, inherits face from blazeface parent.analyze('Start Description:'); if (parent.config.async) { descRes = parent.config.face.description.enabled ? faceres.predict(faces[i], parent.config, i, faces.length) : []; } else { parent.state = 'run:description'; timeStamp = now(); descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].image, parent.config, i, faces.length) : []; parent.perf.embedding = Math.trunc(now() - timeStamp); } parent.analyze('End Description:'); // if async wait for results if (parent.config.async) { [ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]); } parent.analyze('Finish Face:'); // calculate iris distance // iris: array[ center, left, top, right, bottom] if (!parent.config.face.iris.enabled && faces[i]?.annotations?.leftEyeIris && faces[i]?.annotations?.rightEyeIris) { delete faces[i].annotations.leftEyeIris; delete faces[i].annotations.rightEyeIris; } const irisSize = (faces[i].annotations?.leftEyeIris && faces[i].annotations?.rightEyeIris) /* note: average human iris size is 11.7mm */ ? Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) / input.shape[2] : 0; // combine results faceRes.push({ id: i, ...faces[i], age: descRes.age, gender: descRes.gender, genderConfidence: descRes.genderConfidence, embedding: descRes.descriptor, emotion: emotionRes, iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0, rotation, tensor: parent.config.face.detector.return ? faces[i].image?.squeeze() : null, }); // dispose original face tensor faces[i].image?.dispose(); parent.analyze('End Face'); } parent.analyze('End FaceMesh:'); if (parent.config.async) { if (parent.perf.face) delete parent.perf.face; if (parent.perf.age) delete parent.perf.age; if (parent.perf.gender) delete parent.perf.gender; if (parent.perf.emotion) delete parent.perf.emotion; } return faceRes; };