/** * HSE-FaceRes Module * Returns Age, Gender, Descriptor * Implements Face simmilarity function */ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { Tensor, GraphModel } from '../tfjs/types'; import { Config } from '../config'; let model: GraphModel; const last: Array<{ age: number, gender: string, genderScore: number, descriptor: number[], }> = []; let lastCount = 0; let skipped = Number.MAX_SAFE_INTEGER; type DB = Array<{ name: string, source: string, embedding: number[] }>; export async function load(config: Config): Promise { const modelUrl = join(config.modelBasePath, config.face.description.modelPath); if (!model) { // @ts-ignore type mismatch for GraphModel model = await tf.loadGraphModel(modelUrl); if (!model) log('load model failed:', config.face.description.modelPath); else if (config.debug) log('load model:', modelUrl); } else if (config.debug) log('cached model:', modelUrl); return model; } export function similarity(embedding1: Array, embedding2: Array, order = 2): number { if (!embedding1 || !embedding2) return 0; if (embedding1?.length === 0 || embedding2?.length === 0) return 0; if (embedding1?.length !== embedding2?.length) return 0; // general minkowski distance, euclidean distance is limited case where order is 2 const distance = 5.0 * embedding1 .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared .reduce((sum, now) => (sum + now), 0) // sum all distances ** (1 / order); // get root of const res = Math.max(0, 100 - distance) / 100.0; return res; } export function match(embedding: Array, db: DB, threshold = 0) { let best = { similarity: 0, name: '', source: '', embedding: [] as number[] }; if (!embedding || !db || !Array.isArray(embedding) || !Array.isArray(db)) return best; for (const f of db) { if (f.embedding && f.name) { const perc = similarity(embedding, f.embedding); if (perc > threshold && perc > best.similarity) best = { ...f, similarity: perc }; } } return best; } export function enhance(input): Tensor { const image = tf.tidy(() => { // input received from detector is already normalized to 0..1 // input is also assumed to be straightened const tensor = input.image || input.tensor || input; if (!(tensor instanceof tf.Tensor)) return null; // do a tight crop of image and resize it to fit the model const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right // const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test if (!model.inputs[0].shape) return null; // model has no shape so no point continuing const crop = (tensor.shape.length === 3) ? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); /* // just resize to fit the embedding model instead of cropping const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); */ /* // convert to black&white to avoid colorization impact const rgb = [0.2989, 0.5870, 0.1140]; // factors for red/green/blue colors when converting to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html const [red, green, blue] = tf.split(crop, 3, 3); const redNorm = tf.mul(red, rgb[0]); const greenNorm = tf.mul(green, rgb[1]); const blueNorm = tf.mul(blue, rgb[2]); const grayscale = tf.addN([redNorm, greenNorm, blueNorm]); const merge = tf.stack([grayscale, grayscale, grayscale], 3).squeeze(4); */ /* // increase image pseudo-contrast 100% // (or do it per-channel so mean is done on each channel) // (or calculate histogram and do it based on histogram) const mean = merge.mean(); const factor = 2; const contrast = merge.sub(mean).mul(factor).add(mean); */ /* // normalize brightness from 0..1 // silly way of creating pseudo-hdr of image const darken = crop.sub(crop.min()); const lighten = darken.div(darken.max()); */ const norm = tf.mul(crop, 255); return norm; }); return image; } export async function predict(image: Tensor, config: Config, idx, count) { if (!model) return null; if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { skipped++; return last[idx]; } skipped = 0; return new Promise(async (resolve) => { const enhanced = enhance(image); let resT; const obj = { age: 0, gender: 'unknown', genderScore: 0, descriptor: [], }; if (config.face.description.enabled) resT = await model.predict(enhanced); tf.dispose(enhanced); if (resT) { tf.tidy(() => { const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; if (confidence > config.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.genderScore = Math.min(0.99, confidence); } const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; const desc = resT.find((t) => t.shape[1] === 1024); // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8 // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor obj.descriptor = [...desc.dataSync()]; // inside tf.tidy }); resT.forEach((t) => tf.dispose(t)); } last[idx] = obj; lastCount = count; resolve(obj); }); }