mirror of https://github.com/vladmandic/human
88 lines
4.3 KiB
TypeScript
88 lines
4.3 KiB
TypeScript
import { log, join } from '../helpers';
|
|
import * as tf from '../../dist/tfjs.esm.js';
|
|
import * as blazeface from './blazeface';
|
|
import * as facepipeline from './facepipeline';
|
|
import * as coords from './coords';
|
|
|
|
export class MediaPipeFaceMesh {
|
|
facePipeline: any;
|
|
config: any;
|
|
|
|
constructor(blazeFace, blazeMeshModel, irisModel, config) {
|
|
this.facePipeline = new facepipeline.Pipeline(blazeFace, blazeMeshModel, irisModel);
|
|
this.config = config;
|
|
}
|
|
|
|
async estimateFaces(input, config): Promise<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }[]> {
|
|
const predictions = await this.facePipeline.predict(input, config);
|
|
const results: Array<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }> = [];
|
|
for (const prediction of (predictions || [])) {
|
|
if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
|
|
const mesh = prediction.coords ? prediction.coords.arraySync() : [];
|
|
const meshRaw = mesh.map((pt) => [
|
|
pt[0] / input.shape[2],
|
|
pt[1] / input.shape[1],
|
|
pt[2] / this.facePipeline.meshSize,
|
|
]);
|
|
const annotations = {};
|
|
if (mesh && mesh.length > 0) {
|
|
for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
|
|
}
|
|
const box = prediction.box ? [
|
|
Math.max(0, prediction.box.startPoint[0]),
|
|
Math.max(0, prediction.box.startPoint[1]),
|
|
Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
|
|
Math.min(input.shape[1], prediction.box.endPoint[1]) - Math.max(0, prediction.box.startPoint[1]),
|
|
] : 0;
|
|
const boxRaw = prediction.box ? [
|
|
prediction.box.startPoint[0] / input.shape[2],
|
|
prediction.box.startPoint[1] / input.shape[1],
|
|
(prediction.box.endPoint[0] - prediction.box.startPoint[0]) / input.shape[2],
|
|
(prediction.box.endPoint[1] - prediction.box.startPoint[1]) / input.shape[1],
|
|
] : [];
|
|
results.push({
|
|
confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
|
|
boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
|
|
faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
|
|
box,
|
|
boxRaw,
|
|
mesh,
|
|
meshRaw,
|
|
annotations,
|
|
image: prediction.image ? prediction.image.clone() : null,
|
|
});
|
|
if (prediction.coords) prediction.coords.dispose();
|
|
if (prediction.image) prediction.image.dispose();
|
|
}
|
|
return results;
|
|
}
|
|
}
|
|
|
|
let faceModels:[any, any, any] = [null, null, null];
|
|
export async function load(config): Promise<MediaPipeFaceMesh> {
|
|
if ((!faceModels[0] && config.face.enabled) || (!faceModels[1] && config.face.mesh.enabled) || (!faceModels[2] && config.face.iris.enabled)) {
|
|
faceModels = await Promise.all([
|
|
(!faceModels[0] && config.face.enabled) ? blazeface.load(config) : null,
|
|
(!faceModels[1] && config.face.mesh.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.mesh.modelPath), { fromTFHub: config.face.mesh.modelPath.includes('tfhub.dev') }) : null,
|
|
(!faceModels[2] && config.face.iris.enabled) ? tf.loadGraphModel(join(config.modelBasePath, config.face.iris.modelPath), { fromTFHub: config.face.iris.modelPath.includes('tfhub.dev') }) : null,
|
|
]);
|
|
if (config.face.mesh.enabled) {
|
|
if (!faceModels[1] || !faceModels[1].modelUrl) log('load model failed:', config.face.mesh.modelPath);
|
|
else if (config.debug) log('load model:', faceModels[1].modelUrl);
|
|
}
|
|
if (config.face.iris.enabled) {
|
|
if (!faceModels[2] || !faceModels[1].modelUrl) log('load model failed:', config.face.iris.modelPath);
|
|
else if (config.debug) log('load model:', faceModels[2].modelUrl);
|
|
}
|
|
} else if (config.debug) {
|
|
log('cached model:', faceModels[0].model.modelUrl);
|
|
log('cached model:', faceModels[1].modelUrl);
|
|
log('cached model:', faceModels[2].modelUrl);
|
|
}
|
|
const faceMesh = new MediaPipeFaceMesh(faceModels[0], faceModels[1], faceModels[2], config);
|
|
return faceMesh;
|
|
}
|
|
|
|
export const triangulation = coords.TRI468;
|
|
export const uvmap = coords.UV468;
|