human/src/embedding/embedding.ts

import { log } from '../log';
import * as tf from '../../dist/tfjs.esm.js';
import * as profile from '../profile';

let model;

export async function load(config) {
  if (!model) {
    model = await tf.loadGraphModel(config.face.embedding.modelPath);
    if (config.debug) log(`load model: ${config.face.embedding.modelPath.match(/\/(.*)\./)[1]}`);
  }
  return model;
}

export function simmilarity(embedding1, embedding2, order = 2) {
  if (!embedding1 || !embedding2) return 0;
  if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
  if (embedding1?.length !== embedding2?.length) return 0;
  // general minkowski distance
  // euclidean distance is limited case where order is 2
  const distance = embedding1
    .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
    .reduce((sum, now) => (sum + now), 0) // sum all distances
    ** (1 / order); // get root of
  const res = Math.max(Math.trunc(1000 * (1 - (1 * distance))) / 1000, 0);
  return res;
}

export function enhance(input) {
  const image = tf.tidy(() => {
    // input received from detector is already normalized to 0..1
    // input is also assumed to be straightened
    // const data = tf.image.resizeBilinear(input, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); // just resize to fit the embedding model

    // do a tight crop of image and resize it to fit the model
    // maybe offsets are already prepared by face model, if not use empirical values
    const box = input.offsetRaw
      ? [input.offsetRaw] // crop based on face mesh borders
      : [[0.05, 0.15, 0.85, 0.85]]; // fixed crop for top, left, bottom, right
    console.log('BOX', box[0]);
    const tensor = input.image || input.tensor;
    const crop = tensor.shape.length === 3
      ? tf.image.cropAndResize(tensor.expandDims(0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch if missing
      : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);

    // convert to black&white to avoid colorization impact
    const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
    const [red, green, blue] = tf.split(crop, 3, 3);
    const redNorm = tf.mul(red, rgb[0]);
    const greenNorm = tf.mul(green, rgb[1]);
    const blueNorm = tf.mul(blue, rgb[2]);
    const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
    const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4);

    // normalize brightness from 0..1
    const darken = merge.sub(merge.min());
    const lighten = darken.div(darken.max());

    return lighten;
  });
  return image;
}

export async function predict(input, config) {
  if (!model) return null;
  return new Promise(async (resolve) => {
    const image = enhance(input);
    // let data: Array<[]> = [];
    let data: Array<number> = [];
    if (config.face.embedding.enabled) {
      if (!config.profile) {
        const res = await model.predict(image);
        // optional normalize outputs with l2 normalization
        /*
        const scaled = tf.tidy(() => {
          const l2 = res.norm('euclidean');
          const scale = res.div(l2);
          return scale;
        });
        */
        data = res.dataSync();
        // tf.dispose(scaled);
        tf.dispose(res);
      } else {
        const profileData = await tf.profile(() => model.predict({ img_inputs: image }));
        data = [...profileData.result.dataSync()];
        profileData.result.dispose();
        profile.run('emotion', profileData);
      }
    }
    image.dispose();
    resolve(data);
  });
}

/*
git clone https://github.com/becauseofAI/MobileFace
cd MobileFace/MobileFace_Identification
mmconvert --srcFramework mxnet --inputWeight MobileFace_Identification_V3-0000.params --inputNetwork MobileFace_Identification_V3-symbol.json --inputShape 3,112,112 --dstFramework tensorflow --outputModel saved
saved_model_cli show --dir saved/
tensorflowjs_converter --input_format tf_saved_model --output_format tfjs_graph_model --saved_model_tags train saved/ graph/
~/dev/detector/signature.js graph/
2021-03-12 08:25:12 DATA:  created on: 2021-03-12T13:17:11.960Z
2021-03-12 08:25:12 INFO:  graph model: /home/vlado/dev/face/MobileFace/MobileFace_Identification/graph/model.json
2021-03-12 08:25:12 INFO:  size: { unreliable: true, numTensors: 75, numDataBuffers: 75, numBytes: 2183192 }
2021-03-12 08:25:12 INFO:  model inputs based on signature
2021-03-12 08:25:12 INFO:  model outputs based on signature
2021-03-12 08:25:12 DATA:  inputs: [ { name: 'data:0', dtype: 'DT_FLOAT', shape: [ -1, 112, 112, 3, [length]: 4 ] }, [length]: 1 ]
2021-03-12 08:25:12 DATA:  outputs: [ { id: 0, name: 'batchnorm0/add_1:0', dytpe: 'DT_FLOAT', shape: [ -1, 256, [length]: 2 ] }, [length]: 1 ]
*/