human/src/human.js

const tf = require('@tensorflow/tfjs');
const facemesh = require('./facemesh/facemesh.js');
const ssrnet = require('./ssrnet/ssrnet.js');
const emotion = require('./emotion/emotion.js');
const posenet = require('./posenet/posenet.js');
const handpose = require('./handpose/handpose.js');
const defaults = require('../config.js').default;
const app = require('../package.json');

let config;
let state = 'idle';

// object that contains all initialized models
const models = {
  facemesh: null,
  posenet: null,
  handpose: null,
  iris: null,
  age: null,
  gender: null,
  emotion: null,
};

const override = {
  face: { detector: { skipFrames: 0 }, age: { skipFrames: 0 }, emotion: { skipFrames: 0 } },
  hand: { skipFrames: 0 },
};

// helper function: gets elapsed time on both browser and nodejs
const now = () => {
  if (typeof performance !== 'undefined') return performance.now();
  return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000);
};

// helper function: wrapper around console output
const log = (...msg) => {
  // eslint-disable-next-line no-console
  if (msg && config.console) console.log(...msg);
};

// helper function: measure tensor leak
let numTensors = 0;
const analyzeMemoryLeaks = false;
const analyze = (...msg) => {
  if (!analyzeMemoryLeaks) return;
  const current = tf.engine().state.numTensors;
  const previous = numTensors;
  numTensors = current;
  const leaked = current - previous;
  if (leaked !== 0) log(...msg, leaked);
};

// helper function: perform deep merge of multiple objects so it allows full inheriance with overrides
function mergeDeep(...objects) {
  const isObject = (obj) => obj && typeof obj === 'object';
  return objects.reduce((prev, obj) => {
    Object.keys(obj || {}).forEach((key) => {
      const pVal = prev[key];
      const oVal = obj[key];
      if (Array.isArray(pVal) && Array.isArray(oVal)) {
        prev[key] = pVal.concat(...oVal);
      } else if (isObject(pVal) && isObject(oVal)) {
        prev[key] = mergeDeep(pVal, oVal);
      } else {
        prev[key] = oVal;
      }
    });
    return prev;
  }, {});
}

function sanity(input) {
  if (!input) return 'input is not defined';
  if (!(input instanceof tf.Tensor)
      || (tf.ENV.flags.IS_BROWSER
         && (input instanceof ImageData || input instanceof HTMLImageElement || input instanceof HTMLCanvasElement || input instanceof HTMLVideoElement || input instanceof HTMLMediaElement))) {
    const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0));
    if (!width || (width === 0)) return 'input is empty';
  }
  if (tf.ENV.flags.IS_BROWSER && (input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) {
    if (input.readyState && (input.readyState <= 2)) return 'input is not ready';
  }
  if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
    return 'input must be a tensor';
  }
  try {
    tf.getBackend();
  } catch {
    return 'backend not loaded';
  }
  return null;
}

async function load(userConfig) {
  if (userConfig) config = mergeDeep(defaults, userConfig);
  if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face);
  if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body);
  if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand);
  if (config.face.enabled && config.face.age.enabled && !models.age) models.age = await ssrnet.loadAge(config);
  if (config.face.enabled && config.face.gender.enabled && !models.gender) models.gender = await ssrnet.loadGender(config);
  if (config.face.enabled && config.face.emotion.enabled && !models.emotion) models.emotion = await emotion.load(config);
}

function tfImage(input) {
  let image;
  if (input instanceof tf.Tensor) {
    image = tf.clone(input);
  } else {
    const pixels = tf.browser.fromPixels(input);
    const casted = pixels.toFloat();
    image = casted.expandDims(0);
    pixels.dispose();
    casted.dispose();
  }
  return image;
}

async function detect(input, userConfig = {}) {
  state = 'config';
  const perf = {};
  let timeStamp;

  timeStamp = now();
  const shouldOverride = tf.ENV.flags.IS_NODE || (tf.ENV.flags.IS_BROWSER && !((input instanceof HTMLVideoElement) || (input instanceof HTMLMediaElement)));
  config = mergeDeep(defaults, userConfig, shouldOverride ? override : {});
  perf.config = Math.trunc(now() - timeStamp);

  // sanity checks
  timeStamp = now();
  state = 'check';
  const error = sanity(input);
  if (error) {
    log(error, input);
    return { error };
  }
  perf.sanity = Math.trunc(now() - timeStamp);

  // eslint-disable-next-line no-async-promise-executor
  return new Promise(async (resolve) => {
    const timeStart = now();

    // configure backend
    timeStamp = now();
    if (tf.getBackend() !== config.backend) {
      state = 'backend';
      log('Human library setting backend:', config.backend);
      await tf.setBackend(config.backend);
      await tf.ready();
    }
    perf.backend = Math.trunc(now() - timeStamp);

    // check number of loaded models
    const loadedModels = Object.values(models).filter((a) => a).length;
    if (loadedModels === 0) {
      log('Human library starting');
      log('Configuration:', config);
      log('Flags:', tf.ENV.flags);
    }

    // load models if enabled
    timeStamp = now();
    state = 'load';
    await load();
    perf.load = Math.trunc(now() - timeStamp);

    if (config.scoped) tf.engine().startScope();

    analyze('Start Detect:');

    const imageTensor = tfImage(input);

    // run posenet
    state = 'run:body';
    timeStamp = now();
    analyze('Start PoseNet');
    const poseRes = config.body.enabled ? await models.posenet.estimatePoses(imageTensor, config.body) : [];
    analyze('End PoseNet:');
    perf.body = Math.trunc(now() - timeStamp);

    // run handpose
    state = 'run:hand';
    timeStamp = now();
    analyze('Start HandPose:');
    const handRes = config.hand.enabled ? await models.handpose.estimateHands(imageTensor, config.hand) : [];
    analyze('End HandPose:');
    perf.hand = Math.trunc(now() - timeStamp);

    // run facemesh, includes blazeface and iris
    const faceRes = [];
    if (config.face.enabled) {
      state = 'run:face';
      timeStamp = now();
      analyze('Start FaceMesh:');
      const faces = await models.facemesh.estimateFaces(imageTensor, config.face);
      perf.face = Math.trunc(now() - timeStamp);
      for (const face of faces) {
        // is something went wrong, skip the face
        if (!face.image || face.image.isDisposedInternal) {
          log('face object is disposed:', face.image);
          continue;
        }
        // run ssr-net age & gender, inherits face from blazeface
        state = 'run:agegender';
        timeStamp = now();
        const ssrData = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {};
        perf.agegender = Math.trunc(now() - timeStamp);
        // run emotion, inherits face from blazeface
        state = 'run:emotion';
        timeStamp = now();
        const emotionData = config.face.emotion.enabled ? await emotion.predict(face.image, config) : {};
        perf.emotion = Math.trunc(now() - timeStamp);

        // dont need face anymore
        face.image.dispose();
        // calculate iris distance
        // iris: array[ bottom, left, top, right, center ]
        const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
          ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0])
          : 0;
        faceRes.push({
          confidence: face.confidence,
          box: face.box,
          mesh: face.mesh,
          annotations: face.annotations,
          age: ssrData.age,
          gender: ssrData.gender,
          agConfidence: ssrData.confidence,
          emotion: emotionData,
          iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
        });
        analyze('End FaceMesh:');
      }
    }

    imageTensor.dispose();
    state = 'idle';

    if (config.scoped) tf.engine().endScope();
    analyze('End Scope:');

    perf.total = Math.trunc(now() - timeStart);
    resolve({ face: faceRes, body: poseRes, hand: handRes, performance: perf });
  });
}

exports.detect = detect;
exports.defaults = defaults;
exports.config = config;
exports.models = models;
exports.facemesh = facemesh;
exports.ssrnet = ssrnet;
exports.posenet = posenet;
exports.handpose = handpose;
exports.tf = tf;
exports.version = app.version;
exports.state = state;

// Error: Failed to compile fragment shader