human/src/embedding/embedding.ts

import { log } from '../log';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';

// original: https://github.com/sirius-ai/MobileFaceNet_TF
// modified: https://github.com/sirius-ai/MobileFaceNet_TF/issues/46
// download: https://github.com/sirius-ai/MobileFaceNet_TF/files/3551493/FaceMobileNet192_train_false.zip

/* WiP

- Should input box be tightly cropped?
- What is the best input range? (adjust distance scale accordingly)
- How to best normalize output
*/

let model;

export async function load(config) {
  if (!model) {
    model = await tf.loadGraphModel(config.face.embedding.modelPath);
    if (config.debug) log(`load model: ${config.face.embedding.modelPath.match(/\/(.*)\./)[1]}`);
  }
  return model;
}

export function simmilarity(embedding1, embedding2, order = 2) {
  if (!embedding1 || !embedding2) return 0;
  if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
  if (embedding1?.length !== embedding2?.length) return 0;
  // general minkowski distance
  // euclidean distance is limited case where order is 2
  const distance = 50.0 * ((embedding1.map((val, i) => (val - embedding2[i])).reduce((dist, diff) => dist + (diff ** order), 0) ** (1 / order)));
  const res = (Math.trunc(1000 * (1 - (isNaN(distance) ? 1 : distance))) / 1000);
  console.log(distance, res);
  return res;
}

export async function predict(image, config) {
  if (!model) return null;
  return new Promise(async (resolve) => {
    const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); // input is already normalized to 0..1
    // optionally do a tight box crop
    /*
    const box = [[0, 0.2, 0.9, 0.8]]; // top, left, bottom, right
    const resize = tf.image.cropAndResize(image, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
    */
    // debug visualize box
    // const canvas = document.getElementById('compare-canvas');
    // await tf.browser.toPixels(resize.squeeze(), canvas);
    const norm = resize.sub(0.5);
    // optionally normalizes with mean value being at point 0, better than fixed range -0.5..0.5
    /*
    const mean = resize.mean();
    const norm = resize.sub(mean);
    */
    resize.dispose();
    let data: Array<[]> = [];
    if (config.face.embedding.enabled) {
      if (!config.profile) {
        const res = await model.predict({ img_inputs: norm });
        /*
        const scaled = tf.tidy(() => {
          // run l2 normalization on output
          const sqr = res.square();
          const sum = sqr.sum();
          const sqrt = sum.sqrt();
          const l2 = res.div(sqrt);
          // scale outputs
          const range = l2.max().sub(l2.min());
          const scale = l2.mul(2).div(range);
          return scale;
        });
        */
        data = [...res.dataSync()]; // convert object array to standard array
        tf.dispose(res);
      } else {
        const profileData = await tf.profile(() => model.predict({ img_inputs: norm }));
        data = [...profileData.result.dataSync()];
        profileData.result.dispose();
        profile.run('emotion', profileData);
      }
    }
    norm.dispose();
    resolve(data);
  });
}