/** * Loader and Validator for all models used by Human */ import { env } from './util/env'; import { log } from './util/util'; import * as gear from './gear/gear'; import * as ssrnetAge from './gear/ssrnet-age'; import * as ssrnetGender from './gear/ssrnet-gender'; import * as antispoof from './face/antispoof'; import * as blazeface from './face/blazeface'; import * as blazepose from './body/blazepose'; import * as centernet from './object/centernet'; import * as efficientpose from './body/efficientpose'; import * as emotion from './gear/emotion'; import * as facemesh from './face/facemesh'; import * as faceres from './face/faceres'; import * as handpose from './hand/handpose'; import * as handtrack from './hand/handtrack'; import * as iris from './face/iris'; import * as liveness from './face/liveness'; import * as movenet from './body/movenet'; import * as nanodet from './object/nanodet'; import * as posenet from './body/posenet'; import * as segmentation from './segmentation/segmentation'; import type { GraphModel } from './tfjs/types'; import type { Human } from './human'; /** Instances of all possible TFJS Graph Models used by Human * - loaded as needed based on configuration * - initialized explictly with `human.load()` method * - initialized implicity on first call to `human.detect()` * - each model can be `null` if not loaded, instance of `GraphModel` if loaded or `Promise` if loading */ export class Models { ssrnetage: null | GraphModel | Promise = null; gear: null | GraphModel | Promise = null; blazeposedetect: null | GraphModel | Promise = null; blazepose: null | GraphModel | Promise = null; centernet: null | GraphModel | Promise = null; efficientpose: null | GraphModel | Promise = null; embedding: null | GraphModel | Promise = null; emotion: null | GraphModel | Promise = null; facedetect: null | GraphModel | Promise = null; faceiris: null | GraphModel | Promise = null; facemesh: null | GraphModel | Promise = null; faceres: null | GraphModel | Promise = null; ssrnetgender: null | GraphModel | Promise = null; handpose: null | GraphModel | Promise = null; handskeleton: null | GraphModel | Promise = null; handtrack: null | GraphModel | Promise = null; liveness: null | GraphModel | Promise = null; movenet: null | GraphModel | Promise = null; nanodet: null | GraphModel | Promise = null; posenet: null | GraphModel | Promise = null; segmentation: null | GraphModel | Promise = null; antispoof: null | GraphModel | Promise = null; } export function reset(instance: Human): void { // if (instance.config.debug) log('resetting loaded models'); for (const model of Object.keys(instance.models)) instance.models[model] = null; } /** Load method preloads all instance.configured models on-demand */ export async function load(instance: Human): Promise { if (env.initial) reset(instance); if (instance.config.hand.enabled) { // handpose model is a combo that must be loaded as a whole if (!instance.models.handpose && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config); if (!instance.models.handskeleton && instance.config.hand.landmarks && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config); } if (instance.config.body.enabled && !instance.models.blazepose && instance.config.body?.modelPath?.includes('blazepose')) instance.models.blazepose = blazepose.loadPose(instance.config); if (instance.config.body.enabled && !instance.models.blazeposedetect && instance.config.body.detector?.modelPath && instance.config.body?.modelPath?.includes('blazepose')) instance.models.blazeposedetect = blazepose.loadDetect(instance.config); if (instance.config.body.enabled && !instance.models.efficientpose && instance.config.body?.modelPath?.includes('efficientpose')) instance.models.efficientpose = efficientpose.load(instance.config); if (instance.config.body.enabled && !instance.models.movenet && instance.config.body?.modelPath?.includes('movenet')) instance.models.movenet = movenet.load(instance.config); if (instance.config.body.enabled && !instance.models.posenet && instance.config.body?.modelPath?.includes('posenet')) instance.models.posenet = posenet.load(instance.config); if (instance.config.face.enabled && !instance.models.facedetect) instance.models.facedetect = blazeface.load(instance.config); if (instance.config.face.enabled && instance.config.face.antispoof?.enabled && !instance.models.antispoof) instance.models.antispoof = antispoof.load(instance.config); if (instance.config.face.enabled && instance.config.face.liveness?.enabled && !instance.models.liveness) instance.models.liveness = liveness.load(instance.config); if (instance.config.face.enabled && instance.config.face.description?.enabled && !instance.models.faceres) instance.models.faceres = faceres.load(instance.config); if (instance.config.face.enabled && instance.config.face.emotion?.enabled && !instance.models.emotion) instance.models.emotion = emotion.load(instance.config); if (instance.config.face.enabled && instance.config.face.iris?.enabled && !instance.models.faceiris) instance.models.faceiris = iris.load(instance.config); if (instance.config.face.enabled && instance.config.face.mesh?.enabled && !instance.models.facemesh) instance.models.facemesh = facemesh.load(instance.config); if (instance.config.face.enabled && instance.config.face['gear']?.enabled && !instance.models.gear) instance.models.gear = gear.load(instance.config); if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetage) instance.models.ssrnetage = ssrnetAge.load(instance.config); if (instance.config.face.enabled && instance.config.face['ssrnet']?.enabled && !instance.models.ssrnetgender) instance.models.ssrnetgender = ssrnetGender.load(instance.config); if (instance.config.hand.enabled && !instance.models.handtrack && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handtrack = handtrack.loadDetect(instance.config); if (instance.config.hand.enabled && instance.config.hand.landmarks && !instance.models.handskeleton && instance.config.hand.detector?.modelPath?.includes('handtrack')) instance.models.handskeleton = handtrack.loadSkeleton(instance.config); if (instance.config.object.enabled && !instance.models.centernet && instance.config.object?.modelPath?.includes('centernet')) instance.models.centernet = centernet.load(instance.config); if (instance.config.object.enabled && !instance.models.nanodet && instance.config.object?.modelPath?.includes('nanodet')) instance.models.nanodet = nanodet.load(instance.config); if (instance.config.segmentation.enabled && !instance.models.segmentation) instance.models.segmentation = segmentation.load(instance.config); // models are loaded in parallel asynchronously so lets wait until they are actually loaded for await (const model of Object.keys(instance.models)) { if (instance.models[model] && typeof instance.models[model] !== 'undefined') instance.models[model] = await instance.models[model]; } } export async function validate(instance: Human): Promise { interface Op { name: string, category: string, op: string } const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div']; for (const defined of Object.keys(instance.models)) { if (instance.models[defined]) { // check if model is loaded let models: GraphModel[] = []; if (Array.isArray(instance.models[defined])) { models = instance.models[defined] .filter((model) => (model !== null)) .map((model) => ((model && model.executor) ? model : model.model)); } else { models = [instance.models[defined]]; } for (const model of models) { if (!model) { if (instance.config.debug) log('model marked as loaded but not defined:', defined); continue; } const ops: string[] = []; // @ts-ignore // executor is a private method const executor = model?.executor; if (executor && executor.graph.nodes) { for (const kernel of Object.values(executor.graph.nodes)) { const op = (kernel as Op).op.toLowerCase(); if (!ops.includes(op)) ops.push(op); } } else { if (!executor && instance.config.debug) log('model signature not determined:', defined); } const missing: string[] = []; for (const op of ops) { if (!simpleOps.includes(op) // exclude simple ops && !instance.env.kernels.includes(op) // check actual kernel ops && !instance.env.kernels.includes(op.replace('_', '')) // check variation without _ && !instance.env.kernels.includes(op.replace('native', '')) // check standard variation && !instance.env.kernels.includes(op.replace('v2', ''))) { // check non-versioned variation missing.push(op); } } // log('model validation ops:', defined, ops); if (missing.length > 0 && instance.config.debug) log('model validation:', defined, missing); } } } }