/** * EfficientPose model implementation * * Based on: [**BecauseofAI MobileFace**](https://github.com/becauseofAI/MobileFace) * * Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis */ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import type { Tensor, GraphModel } from '../tfjs/types'; import { env } from '../util/env'; let model: GraphModel | null; export async function load(config) { const modelUrl = join(config.modelBasePath, config.face.embedding.modelPath); if (env.initial) model = null; if (!model) { model = await tf.loadGraphModel(modelUrl) as unknown as GraphModel; if (!model) log('load model failed:', config.face.embedding.modelPath); else if (config.debug) log('load model:', modelUrl); } else if (config.debug) log('cached model:', modelUrl); return model; } export function enhance(input): Tensor { const image = tf.tidy(() => { // input received from detector is already normalized to 0..1 // input is also assumed to be straightened // const data = tf.image.resizeBilinear(input, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); // just resize to fit the embedding model // do a tight crop of image and resize it to fit the model const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right const tensor = input.image || input.tensor; if (!(tensor instanceof tf.Tensor)) return null; if (!model || !model.inputs || !model.inputs[0].shape) return null; const crop = (tensor.shape.length === 3) ? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); // convert to black&white to avoid colorization impact const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html const [red, green, blue] = tf.split(crop, 3, 3); const redNorm = tf.mul(red, rgb[0]); const greenNorm = tf.mul(green, rgb[1]); const blueNorm = tf.mul(blue, rgb[2]); const grayscale = tf.addN([redNorm, greenNorm, blueNorm]); const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4); /* // optional increase image contrast // or do it per-channel so mean is done on each channel // or do it based on histogram const mean = merge.mean(); const factor = 5; const contrast = merge.sub(mean).mul(factor).add(mean); */ // normalize brightness from 0..1 const darken = tf.sub(merge, merge.min()); const lighten = tf.div(darken, darken.max()); return lighten; }); return image; } export async function predict(input, config): Promise { if (!model) return []; return new Promise(async (resolve) => { // let data: Array<[]> = []; let data: Array = []; if (config.face.embedding.enabled) { const image = enhance(input); const dataT = model?.execute(image) as Tensor; /* const dataT = tf.tidy(() => { /* // if needed convert from NHWC to NCHW const nchw = image.transpose([3, 0, 1, 2]); const res = model.execute(image); // optionally do it twice with flipped image and average results const res1 = model.execute(image); const flipped = tf.image.flipLeftRight(image); const res2 = model.execute(flipped); const merge = tf.stack([res1, res2], 2).squeeze(); const res = reshape.logSumExp(1); // optional normalize outputs with l2 normalization const scaled = tf.tidy(() => { const l2 = res.norm('euclidean'); const scale = res.div(l2); return scale; }); // optional reduce feature vector complexity const reshape = tf.reshape(res, [128, 2]); // split 256 vectors into 128 x 2 const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it return reduce; }); */ const output = await dataT.data(); data = Array.from(output); // convert typed array to simple array tf.dispose(dataT); tf.dispose(image); } resolve(data); }); }