diff --git a/CHANGELOG.md b/CHANGELOG.md index 961d6cb2..259afcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human - Version: **2.8.1** + Version: **2.9.0** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,7 +9,7 @@ ## Changelog -### **HEAD -> main** 2022/07/14 mandic00@live.com +### **HEAD -> main** 2022/07/16 mandic00@live.com - placeholder for face contours - improve face compare in main demo diff --git a/build.js b/build.js index c25ac27e..f9459599 100644 --- a/build.js +++ b/build.js @@ -2,12 +2,11 @@ const fs = require('fs'); const log = require('@vladmandic/pilogger'); const Build = require('@vladmandic/build').Build; const APIExtractor = require('@microsoft/api-extractor'); +const tf = require('@tensorflow/tfjs-node'); +const package = require('./package.json'); -function copy(src, dst) { - if (!fs.existsSync(src)) return; - const buffer = fs.readFileSync(src); - fs.writeFileSync(dst, buffer); -} +const modelsDir = '../human-models/models'; +const modelsOut = 'models/models.json'; // eslint-disable-next-line @typescript-eslint/no-unused-vars const apiExtractorIgnoreList = [ @@ -22,7 +21,46 @@ const apiExtractorIgnoreList = [ 'tsdoc-unnecessary-backslash', ]; +function copy(src, dst) { + if (!fs.existsSync(src)) return; + const buffer = fs.readFileSync(src); + fs.writeFileSync(dst, buffer); +} + +async function analyzeModels() { + log.info('Analyze:', { modelsDir, modelsOut }); + let totalSize = 0; + const models = {}; + let dir; + try { + dir = fs.readdirSync(modelsDir); + } catch { + log.warn('Cannot enumerate:', modelsDir); + } + if (!dir || dir.length === 0) { + log.warn('No models found:', modelsDir); + return; + } + for (const f of dir) { + if (!f.endsWith('.json')) continue; + const url = `file://${modelsDir}/${f}`; + const model = new tf.GraphModel(url); // create model prototype and decide if load from cache or from original modelurl + model.findIOHandler(); + const artifacts = await model.handler.load(); + const size = artifacts?.weightData?.byteLength || 0; + totalSize += size; + const name = f.replace('.json', ''); + models[name] = size; + } + const json = JSON.stringify(models, null, 2); + fs.writeFileSync(modelsOut, json); + log.state('Models:', { count: Object.keys(models).length, totalSize }); +} + async function main() { + log.data('Build', { name: package.name, version: package.version }); + // generate model signature + await analyzeModels(); // run production build const build = new Build(); await build.run('production'); diff --git a/models/models.json b/models/models.json new file mode 100644 index 00000000..4e46b1b4 --- /dev/null +++ b/models/models.json @@ -0,0 +1,44 @@ +{ + "age": 161240, + "antispoof": 853098, + "blazeface-back": 538928, + "blazeface-front": 402048, + "blazeface": 538928, + "blazepose-detector2d": 7499400, + "blazepose-detector3d": 5928856, + "blazepose-full": 6338290, + "blazepose-heavy": 27501554, + "blazepose-lite": 2725490, + "efficientpose": 5651240, + "emotion": 820516, + "faceboxes": 2013002, + "facemesh-attention-alt": 2387598, + "facemesh-attention": 2382414, + "facemesh-detection-full": 1026192, + "facemesh-detection-short": 201268, + "facemesh-orig": 2955780, + "facemesh": 1477958, + "faceres-deep": 13957620, + "faceres": 6978814, + "gear": 1498916, + "gender-ssrnet-imdb": 161236, + "gender": 201808, + "handdetect": 3515612, + "handlandmark-full": 5431368, + "handlandmark-lite": 2023432, + "handlandmark-sparse": 5286322, + "handskeleton": 5502280, + "handtrack": 2964837, + "iris": 2599092, + "liveness": 592976, + "mb3-centernet": 4030290, + "meet": 372228, + "mobileface": 2183192, + "mobilefacenet": 5171976, + "movenet-lightning": 4650216, + "movenet-multipose": 9448838, + "movenet-thunder": 12477112, + "nanodet": 7574558, + "posenet": 5032780, + "selfie": 212886 +} \ No newline at end of file diff --git a/package.json b/package.json index fbe1391c..da515c24 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vladmandic/human", - "version": "2.8.1", + "version": "2.9.0", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "sideEffects": false, "main": "dist/human.node.js", @@ -69,7 +69,7 @@ "@tensorflow/tfjs-layers": "^3.18.0", "@tensorflow/tfjs-node": "^3.18.0", "@tensorflow/tfjs-node-gpu": "^3.18.0", - "@types/node": "^18.0.4", + "@types/node": "^18.0.6", "@types/offscreencanvas": "^2019.7.0", "@typescript-eslint/eslint-plugin": "^5.30.6", "@typescript-eslint/parser": "^5.30.6", @@ -77,7 +77,7 @@ "@vladmandic/pilogger": "^0.4.5", "@vladmandic/tfjs": "github:vladmandic/tfjs", "esbuild": "^0.14.49", - "eslint": "8.19.0", + "eslint": "8.20.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-html": "^6.2.0", "eslint-plugin-import": "^2.26.0", @@ -88,7 +88,7 @@ "rimraf": "^3.0.2", "seedrandom": "^3.0.5", "tslib": "^2.4.0", - "typedoc": "0.23.7", + "typedoc": "0.23.8", "typescript": "4.7.4" } } diff --git a/src/exports.ts b/src/exports.ts index 2c242b52..1f1a61a3 100644 --- a/src/exports.ts +++ b/src/exports.ts @@ -39,3 +39,7 @@ export type ImageObjects = ImageData | ImageBitmap export type ExternalCanvas = typeof env.Canvas; /** Defines all possible input types for **Human** detection */ export type Input = Tensor | AnyCanvas | AnyImage | AnyVideo | ImageObjects | ExternalCanvas; +/** Defines model stats */ +export type { ModelStats } from './models'; +/** Defines individual model sizes */ +export type { ModelInfo } from './tfjs/load'; diff --git a/src/human.ts b/src/human.ts index 9aed5e4f..2873e276 100644 --- a/src/human.ts +++ b/src/human.ts @@ -37,7 +37,7 @@ import * as posenet from './body/posenet'; import * as segmentation from './segmentation/segmentation'; import * as warmups from './warmup'; // type definitions -import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas } from './exports'; +import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas, ModelStats } from './exports'; // type exports export * from './exports'; @@ -330,7 +330,7 @@ export class Human { } /** get model loading/loaded stats */ - getModelStats() { return models.getModelStats(); } + getModelStats(): ModelStats { return models.getModelStats(this); } /** Warmup method pre-initializes all configured models for faster inference * - can take significant time on startup diff --git a/src/models.ts b/src/models.ts index e776810a..4e6fa0c8 100644 --- a/src/models.ts +++ b/src/models.ts @@ -24,7 +24,7 @@ import * as movenet from './body/movenet'; import * as nanodet from './object/nanodet'; import * as posenet from './body/posenet'; import * as segmentation from './segmentation/segmentation'; -import { modelStats } from './tfjs/load'; +import { modelStats, ModelInfo } from './tfjs/load'; import type { GraphModel } from './tfjs/types'; import type { Human } from './human'; @@ -59,14 +59,36 @@ export class Models { antispoof: null | GraphModel | Promise = null; } -export const getModelStats = () => { - let sizeFromManifest = 0; - let sizeWeights = 0; +export type ModelStats = { + numLoadedModels: number, + numEnabledModels: undefined, + numDefinedModels: number, + totalSizeFromManifest: number, + totalSizeWeights: number, + totalSizeLoading: number, + totalSizeEnabled: undefined, + modelStats: ModelInfo[], +} + +export const getModelStats = (instance: Human): ModelStats => { + let totalSizeFromManifest = 0; + let totalSizeWeights = 0; + let totalSizeLoading = 0; for (const m of Object.values(modelStats)) { - sizeFromManifest += m.manifest; - sizeWeights += m.weights; + totalSizeFromManifest += m.sizeFromManifest; + totalSizeWeights += m.sizeLoadedWeights; + totalSizeLoading += m.sizeDesired; } - return { numLoadedModels: Object.values(modelStats).length, sizeFromManifest, sizeWeights }; + return { + numLoadedModels: Object.values(modelStats).length, + numEnabledModels: undefined, + numDefinedModels: Object.keys(instance.models).length, + totalSizeFromManifest, + totalSizeWeights, + totalSizeLoading, + totalSizeEnabled: undefined, + modelStats: Object.values(modelStats), + }; }; export function reset(instance: Human): void { diff --git a/src/tfjs/load.ts b/src/tfjs/load.ts index def5ca67..807ee006 100644 --- a/src/tfjs/load.ts +++ b/src/tfjs/load.ts @@ -2,6 +2,7 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import type { GraphModel } from './types'; import type { Config } from '../config'; +import * as modelsDefs from '../../models/models.json'; const options = { cacheModels: true, @@ -11,14 +12,15 @@ const options = { modelBasePath: '', }; -type ModelStats = { +export type ModelInfo = { name: string, - cached: boolean, - manifest: number, - weights: number, + inCache: boolean, + sizeDesired: number, + sizeFromManifest: number, + sizeLoadedWeights: number, } -export const modelStats: Record = {}; +export const modelStats: Record = {}; async function httpHandler(url, init?): Promise { if (options.debug) log('load model fetch:', url, init); @@ -34,14 +36,15 @@ export function setModelLoadOptions(config: Config) { export async function loadModel(modelPath: string | undefined): Promise { let modelUrl = join(options.modelBasePath, modelPath || ''); if (!modelUrl.toLowerCase().endsWith('.json')) modelUrl += '.json'; - const modelPathSegments = modelUrl.split('/'); + const modelPathSegments = modelUrl.includes('/') ? modelUrl.split('/') : modelUrl.split('\\'); const shortModelName = modelPathSegments[modelPathSegments.length - 1].replace('.json', ''); const cachedModelName = 'indexeddb://' + shortModelName; // generate short model name for cache modelStats[shortModelName] = { name: shortModelName, - manifest: 0, - weights: 0, - cached: false, + sizeFromManifest: 0, + sizeLoadedWeights: 0, + sizeDesired: modelsDefs[shortModelName], + inCache: false, }; options.cacheSupported = (typeof window !== 'undefined') && (typeof window.localStorage !== 'undefined') && (typeof window.indexedDB !== 'undefined'); // check if running in browser and if indexedb is available let cachedModels = {}; @@ -50,9 +53,9 @@ export async function loadModel(modelPath: string | undefined): Promise httpHandler(url, init) }; - const model: GraphModel = new tf.GraphModel(modelStats[shortModelName].cached ? cachedModelName : modelUrl, tfLoadOptions) as unknown as GraphModel; // create model prototype and decide if load from cache or from original modelurl + const model: GraphModel = new tf.GraphModel(modelStats[shortModelName].inCache ? cachedModelName : modelUrl, tfLoadOptions) as unknown as GraphModel; // create model prototype and decide if load from cache or from original modelurl let loaded = false; try { // @ts-ignore private function @@ -60,16 +63,16 @@ export async function loadModel(modelPath: string | undefined): Promise { - numLoadedModels: number; - sizeFromManifest: number; - sizeWeights: number; -}; +declare const getModelStats: (instance: Human) => ModelStats; declare const getSaveHandlers: (url: string | string[]) => IOHandler[]; @@ -1318,11 +1314,7 @@ declare class Human { */ next(result?: Result): Result; /** get model loading/loaded stats */ - getModelStats(): { - numLoadedModels: number; - sizeFromManifest: number; - sizeWeights: number; - }; + getModelStats(): ModelStats; /** Warmup method pre-initializes all configured models for faster inference * - can take significant time on startup * - only used for `webgl` and `humangl` backends @@ -1740,6 +1732,14 @@ declare interface ModelArtifactsInfo { weightDataBytes?: number; } +export declare type ModelInfo = { + name: string; + inCache: boolean; + sizeDesired: number; + sizeFromManifest: number; + sizeLoadedWeights: number; +}; + /** * The on-disk format of the `model.json` file. * @@ -1852,11 +1852,23 @@ declare namespace models { load, validate, Models, + ModelStats, getModelStats } } export { models } +export declare type ModelStats = { + numLoadedModels: number; + numEnabledModels: undefined; + numDefinedModels: number; + totalSizeFromManifest: number; + totalSizeWeights: number; + totalSizeLoading: number; + totalSizeEnabled: undefined; + modelStats: ModelInfo[]; +}; + /** * An interface for the manager of a model store. *