human/src/faceall.ts

import { log, now } from './helpers';
import * as tf from '../dist/tfjs.esm.js';
import * as age from './age/age';
import * as gender from './gender/gender';
import * as emotion from './emotion/emotion';
import * as embedding from './embedding/embedding';
import * as faceres from './faceres/faceres';

type Tensor = typeof tf.Tensor;

const calculateFaceAngle = (face, image_size): { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] } => {
  // normalize vector
  function normalize(v) {
    const length = Math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]);
    v[0] /= length;
    v[1] /= length;
    v[2] /= length;
    return v;
  }

  // vector subtraction (a - b)
  function subVectors(a, b) {
    const x = a[0] - b[0];
    const y = a[1] - b[1];
    const z = a[2] - b[2];
    return [x, y, z];
  }

  // vector cross product (a x b)
  function crossVectors(a, b) {
    const x = a[1] * b[2] - a[2] * b[1];
    const y = a[2] * b[0] - a[0] * b[2];
    const z = a[0] * b[1] - a[1] * b[0];
    return [x, y, z];
  }

  // 3x3 rotation matrix to Euler angles
  // https://www.geometrictools.com/Documentation/EulerAngles.pdf
  function rotationMatrixToEulerAngle(r) {
    // r01 is not used yet (no-unused-vars)
    // eslint-disable-next-line
    const [ r00, r01, r02, r10, r11, r12, r20, r21, r22 ] = r;
    const pi = Math.PI;

    let thetaX; let thetaY; let thetaZ;

    // YZX
    if (r10 < 1) {
      if (r10 > -1) {
        thetaZ = Math.asin(r10);
        thetaY = Math.atan2(-r20, r00);
        thetaX = Math.atan2(-r12, r11);
      } else {
        thetaZ = -pi / 2;
        thetaY = -Math.atan2(r21, r22);
        thetaX = 0;
      }
    } else {
      thetaZ = pi / 2;
      thetaY = Math.atan2(r21, r22);
      thetaX = 0;
    }

    // compensate Y rotation (from XYZ rotation order routine) which is not accurate and too small in YZX calculation
    if (r02 < 1) {
      if (r02 > -1) {
        thetaY = Math.asin(r02);
      } else {
        thetaY = -pi / 2;
      }
    } else {
      thetaY = pi / 2;
    }

    // pitch, yaw, row
    return [-thetaX, -thetaY, -thetaZ];
  }

  const mesh = face.meshRaw;
  if (!mesh || mesh.length < 300) return { pitch: 0, yaw: 0, row: 0, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1] };

  const size = Math.max(face.boxRaw[2] * image_size[0], face.boxRaw[3] * image_size[1]) / 1.5;
  // top, bottom, left, right
  const pts = [mesh[10], mesh[152], mesh[234], mesh[454]].map((pt) => [
    // make the xyz coordinates proportional, independent of the image/box size
    pt[0] * image_size[0] / size,
    pt[1] * image_size[1] / size,
    pt[2],
  ]);

  const y_axis = normalize(subVectors(pts[1], pts[0]));
  let x_axis = normalize(subVectors(pts[3], pts[2]));
  const z_axis = normalize(crossVectors(x_axis, y_axis));
  // adjust x_axis to make sure that all axes are perpendicular to each other
  x_axis = crossVectors(y_axis, z_axis);

  // Rotation Matrix from Axis Vectors - http://renderdan.blogspot.com/2006/05/rotation-matrix-from-axis-vectors.html
  // 3x3 rotation matrix is flatten to array in row-major order. Note that the rotation represented by this matrix is inverted.
  const r_matrix = [
    x_axis[0], x_axis[1], x_axis[2],
    y_axis[0], y_axis[1], y_axis[2],
    z_axis[0], z_axis[1], z_axis[2],
  ];

  const [pitch, yaw, row] = rotationMatrixToEulerAngle(r_matrix);

  return {
    pitch,
    yaw,
    row,
    matrix: r_matrix,
  };
};

export const detectFace = async (parent, input): Promise<any> => {
  // run facemesh, includes blazeface and iris
  // eslint-disable-next-line no-async-promise-executor
  let timeStamp;
  let ageRes;
  let genderRes;
  let emotionRes;
  let embeddingRes;
  let descRes;
  const faceRes: Array<{
      confidence: number,
      boxConfidence: number,
      faceConfidence: number,
      box: [number, number, number, number],
      mesh: Array<[number, number, number]>
      meshRaw: Array<[number, number, number]>
      boxRaw: [number, number, number, number],
      annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
      age: number,
      gender: string,
      genderConfidence: number,
      emotion: string,
      embedding: number[],
      iris: number,
      angle: { pitch: number, yaw: number, row: number, matrix: [number, number, number, number, number, number, number, number, number] },
      tensor: Tensor,
    }> = [];
  parent.state = 'run:face';
  timeStamp = now();
  const faces = await parent.models.face?.estimateFaces(input, parent.config);
  parent.perf.face = Math.trunc(now() - timeStamp);
  if (!faces) return [];
  for (const face of faces) {
    parent.analyze('Get Face');

    // is something went wrong, skip the face
    if (!face.image || face.image.isDisposedInternal) {
      log('Face object is disposed:', face.image);
      continue;
    }

    const angle = calculateFaceAngle(face, [input.shape[2], input.shape[1]]);

    // run age, inherits face from blazeface
    parent.analyze('Start Age:');
    if (parent.config.async) {
      ageRes = parent.config.face.age.enabled ? age.predict(face.image, parent.config) : {};
    } else {
      parent.state = 'run:age';
      timeStamp = now();
      ageRes = parent.config.face.age.enabled ? await age.predict(face.image, parent.config) : {};
      parent.perf.age = Math.trunc(now() - timeStamp);
    }

    // run gender, inherits face from blazeface
    parent.analyze('Start Gender:');
    if (parent.config.async) {
      genderRes = parent.config.face.gender.enabled ? gender.predict(face.image, parent.config) : {};
    } else {
      parent.state = 'run:gender';
      timeStamp = now();
      genderRes = parent.config.face.gender.enabled ? await gender.predict(face.image, parent.config) : {};
      parent.perf.gender = Math.trunc(now() - timeStamp);
    }

    // run emotion, inherits face from blazeface
    parent.analyze('Start Emotion:');
    if (parent.config.async) {
      emotionRes = parent.config.face.emotion.enabled ? emotion.predict(face.image, parent.config) : {};
    } else {
      parent.state = 'run:emotion';
      timeStamp = now();
      emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(face.image, parent.config) : {};
      parent.perf.emotion = Math.trunc(now() - timeStamp);
    }
    parent.analyze('End Emotion:');

    // run emotion, inherits face from blazeface
    parent.analyze('Start Embedding:');
    if (parent.config.async) {
      embeddingRes = parent.config.face.embedding.enabled ? embedding.predict(face, parent.config) : [];
    } else {
      parent.state = 'run:embedding';
      timeStamp = now();
      embeddingRes = parent.config.face.embedding.enabled ? await embedding.predict(face, parent.config) : [];
      parent.perf.embedding = Math.trunc(now() - timeStamp);
    }
    parent.analyze('End Embedding:');

    // run emotion, inherits face from blazeface
    parent.analyze('Start Description:');
    if (parent.config.async) {
      descRes = parent.config.face.description.enabled ? faceres.predict(face, parent.config) : [];
    } else {
      parent.state = 'run:description';
      timeStamp = now();
      descRes = parent.config.face.description.enabled ? await faceres.predict(face.image, parent.config) : [];
      parent.perf.embedding = Math.trunc(now() - timeStamp);
    }
    parent.analyze('End Description:');

    // if async wait for results
    if (parent.config.async) {
      [ageRes, genderRes, emotionRes, embeddingRes, descRes] = await Promise.all([ageRes, genderRes, emotionRes, embeddingRes, descRes]);
    }

    parent.analyze('Finish Face:');

    // calculate iris distance
    // iris: array[ center, left, top, right, bottom]
    if (!parent.config.face.iris.enabled && face?.annotations?.leftEyeIris && face?.annotations?.rightEyeIris) {
      delete face.annotations.leftEyeIris;
      delete face.annotations.rightEyeIris;
    }
    const irisSize = (face.annotations?.leftEyeIris && face.annotations?.rightEyeIris)
    /* average human iris size is 11.7mm */
      ? 11.7 * Math.max(Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]), Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]))
      : 0;

    // combine results
    faceRes.push({
      ...face,
      age: descRes.age || ageRes.age,
      gender: descRes.gender || genderRes.gender,
      genderConfidence: descRes.genderConfidence || genderRes.confidence,
      embedding: descRes.descriptor || embeddingRes,
      emotion: emotionRes,
      iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0,
      angle,
      tensor: parent.config.face.detector.return ? face.image?.squeeze() : null,
    });
    // dispose original face tensor
    face.image?.dispose();

    parent.analyze('End Face');
  }
  parent.analyze('End FaceMesh:');
  if (parent.config.async) {
    if (parent.perf.face) delete parent.perf.face;
    if (parent.perf.age) delete parent.perf.age;
    if (parent.perf.gender) delete parent.perf.gender;
    if (parent.perf.emotion) delete parent.perf.emotion;
  }
  return faceRes;
};