ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

beta for upcoming major release

pull/356/head
Vladimir Mandic 2022-07-17 21:31:08 -04:00
parent b5390363b5
commit 85656cdef5
9 changed files with 170 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human
Version: **2.8.1**
Version: **2.9.0**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,7 +9,7 @@
## Changelog
### **HEAD -> main** 2022/07/14 mandic00@live.com
### **HEAD -> main** 2022/07/16 mandic00@live.com
- placeholder for face contours
- improve face compare in main demo

48
build.js
View File

@ -2,12 +2,11 @@ const fs = require('fs');
const log = require('@vladmandic/pilogger');
const Build = require('@vladmandic/build').Build;
const APIExtractor = require('@microsoft/api-extractor');
const tf = require('@tensorflow/tfjs-node');
const package = require('./package.json');
function copy(src, dst) {
if (!fs.existsSync(src)) return;
const buffer = fs.readFileSync(src);
fs.writeFileSync(dst, buffer);
}
const modelsDir = '../human-models/models';
const modelsOut = 'models/models.json';
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const apiExtractorIgnoreList = [
@ -22,7 +21,46 @@ const apiExtractorIgnoreList = [
'tsdoc-unnecessary-backslash',
];
function copy(src, dst) {
if (!fs.existsSync(src)) return;
const buffer = fs.readFileSync(src);
fs.writeFileSync(dst, buffer);
}
async function analyzeModels() {
log.info('Analyze:', { modelsDir, modelsOut });
let totalSize = 0;
const models = {};
let dir;
try {
dir = fs.readdirSync(modelsDir);
} catch {
log.warn('Cannot enumerate:', modelsDir);
}
if (!dir || dir.length === 0) {
log.warn('No models found:', modelsDir);
return;
}
for (const f of dir) {
if (!f.endsWith('.json')) continue;
const url = `file://${modelsDir}/${f}`;
const model = new tf.GraphModel(url); // create model prototype and decide if load from cache or from original modelurl
model.findIOHandler();
const artifacts = await model.handler.load();
const size = artifacts?.weightData?.byteLength || 0;
totalSize += size;
const name = f.replace('.json', '');
models[name] = size;
}
const json = JSON.stringify(models, null, 2);
fs.writeFileSync(modelsOut, json);
log.state('Models:', { count: Object.keys(models).length, totalSize });
}
async function main() {
log.data('Build', { name: package.name, version: package.version });
// generate model signature
await analyzeModels();
// run production build
const build = new Build();
await build.run('production');

44
models/models.json Normal file
View File

@ -0,0 +1,44 @@
{
"age": 161240,
"antispoof": 853098,
"blazeface-back": 538928,
"blazeface-front": 402048,
"blazeface": 538928,
"blazepose-detector2d": 7499400,
"blazepose-detector3d": 5928856,
"blazepose-full": 6338290,
"blazepose-heavy": 27501554,
"blazepose-lite": 2725490,
"efficientpose": 5651240,
"emotion": 820516,
"faceboxes": 2013002,
"facemesh-attention-alt": 2387598,
"facemesh-attention": 2382414,
"facemesh-detection-full": 1026192,
"facemesh-detection-short": 201268,
"facemesh-orig": 2955780,
"facemesh": 1477958,
"faceres-deep": 13957620,
"faceres": 6978814,
"gear": 1498916,
"gender-ssrnet-imdb": 161236,
"gender": 201808,
"handdetect": 3515612,
"handlandmark-full": 5431368,
"handlandmark-lite": 2023432,
"handlandmark-sparse": 5286322,
"handskeleton": 5502280,
"handtrack": 2964837,
"iris": 2599092,
"liveness": 592976,
"mb3-centernet": 4030290,
"meet": 372228,
"mobileface": 2183192,
"mobilefacenet": 5171976,
"movenet-lightning": 4650216,
"movenet-multipose": 9448838,
"movenet-thunder": 12477112,
"nanodet": 7574558,
"posenet": 5032780,
"selfie": 212886
}

8
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@vladmandic/human",
"version": "2.8.1",
"version": "2.9.0",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false,
"main": "dist/human.node.js",
@ -69,7 +69,7 @@
"@tensorflow/tfjs-layers": "^3.18.0",
"@tensorflow/tfjs-node": "^3.18.0",
"@tensorflow/tfjs-node-gpu": "^3.18.0",
"@types/node": "^18.0.4",
"@types/node": "^18.0.6",
"@types/offscreencanvas": "^2019.7.0",
"@typescript-eslint/eslint-plugin": "^5.30.6",
"@typescript-eslint/parser": "^5.30.6",
@ -77,7 +77,7 @@
"@vladmandic/pilogger": "^0.4.5",
"@vladmandic/tfjs": "github:vladmandic/tfjs",
"esbuild": "^0.14.49",
"eslint": "8.19.0",
"eslint": "8.20.0",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-plugin-html": "^6.2.0",
"eslint-plugin-import": "^2.26.0",
@ -88,7 +88,7 @@
"rimraf": "^3.0.2",
"seedrandom": "^3.0.5",
"tslib": "^2.4.0",
"typedoc": "0.23.7",
"typedoc": "0.23.8",
"typescript": "4.7.4"
}
}

4
src/exports.ts
View File

@ -39,3 +39,7 @@ export type ImageObjects = ImageData | ImageBitmap
export type ExternalCanvas = typeof env.Canvas;
/** Defines all possible input types for **Human** detection */
export type Input = Tensor | AnyCanvas | AnyImage | AnyVideo | ImageObjects | ExternalCanvas;
/** Defines model stats */
export type { ModelStats } from './models';
/** Defines individual model sizes */
export type { ModelInfo } from './tfjs/load';

4
src/human.ts
View File

@ -37,7 +37,7 @@ import * as posenet from './body/posenet';
import * as segmentation from './segmentation/segmentation';
import * as warmups from './warmup';
// type definitions
import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas } from './exports';
import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas, ModelStats } from './exports';
// type exports
export * from './exports';
@ -330,7 +330,7 @@ export class Human {
}
/** get model loading/loaded stats */
getModelStats() { return models.getModelStats(); }
getModelStats(): ModelStats { return models.getModelStats(this); }
/** Warmup method pre-initializes all configured models for faster inference
* - can take significant time on startup

36
src/models.ts
View File

@ -24,7 +24,7 @@ import * as movenet from './body/movenet';
import * as nanodet from './object/nanodet';
import * as posenet from './body/posenet';
import * as segmentation from './segmentation/segmentation';
import { modelStats } from './tfjs/load';
import { modelStats, ModelInfo } from './tfjs/load';
import type { GraphModel } from './tfjs/types';
import type { Human } from './human';
@ -59,14 +59,36 @@ export class Models {
antispoof: null | GraphModel | Promise<GraphModel> = null;
}
export const getModelStats = () => {
let sizeFromManifest = 0;
let sizeWeights = 0;
export type ModelStats = {
numLoadedModels: number,
numEnabledModels: undefined,
numDefinedModels: number,
totalSizeFromManifest: number,
totalSizeWeights: number,
totalSizeLoading: number,
totalSizeEnabled: undefined,
modelStats: ModelInfo[],
}
export const getModelStats = (instance: Human): ModelStats => {
let totalSizeFromManifest = 0;
let totalSizeWeights = 0;
let totalSizeLoading = 0;
for (const m of Object.values(modelStats)) {
sizeFromManifest += m.manifest;
sizeWeights += m.weights;
totalSizeFromManifest += m.sizeFromManifest;
totalSizeWeights += m.sizeLoadedWeights;
totalSizeLoading += m.sizeDesired;
}
return { numLoadedModels: Object.values(modelStats).length, sizeFromManifest, sizeWeights };
return {
numLoadedModels: Object.values(modelStats).length,
numEnabledModels: undefined,
numDefinedModels: Object.keys(instance.models).length,
totalSizeFromManifest,
totalSizeWeights,
totalSizeLoading,
totalSizeEnabled: undefined,
modelStats: Object.values(modelStats),
};
};
export function reset(instance: Human): void {

33
src/tfjs/load.ts
View File

@ -2,6 +2,7 @@ import { log, join } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import type { GraphModel } from './types';
import type { Config } from '../config';
import * as modelsDefs from '../../models/models.json';
const options = {
cacheModels: true,
@ -11,14 +12,15 @@ const options = {
modelBasePath: '',
};
type ModelStats = {
export type ModelInfo = {
name: string,
cached: boolean,
manifest: number,
weights: number,
inCache: boolean,
sizeDesired: number,
sizeFromManifest: number,
sizeLoadedWeights: number,
}
export const modelStats: Record<string, ModelStats> = {};
export const modelStats: Record<string, ModelInfo> = {};
async function httpHandler(url, init?): Promise<Response | null> {
if (options.debug) log('load model fetch:', url, init);
@ -34,14 +36,15 @@ export function setModelLoadOptions(config: Config) {
export async function loadModel(modelPath: string | undefined): Promise<GraphModel> {
let modelUrl = join(options.modelBasePath, modelPath || '');
if (!modelUrl.toLowerCase().endsWith('.json')) modelUrl += '.json';
const modelPathSegments = modelUrl.split('/');
const modelPathSegments = modelUrl.includes('/') ? modelUrl.split('/') : modelUrl.split('\\');
const shortModelName = modelPathSegments[modelPathSegments.length - 1].replace('.json', '');
const cachedModelName = 'indexeddb://' + shortModelName; // generate short model name for cache
modelStats[shortModelName] = {
name: shortModelName,
manifest: 0,
weights: 0,
cached: false,
sizeFromManifest: 0,
sizeLoadedWeights: 0,
sizeDesired: modelsDefs[shortModelName],
inCache: false,
};
options.cacheSupported = (typeof window !== 'undefined') && (typeof window.localStorage !== 'undefined') && (typeof window.indexedDB !== 'undefined'); // check if running in browser and if indexedb is available
let cachedModels = {};
@ -50,9 +53,9 @@ export async function loadModel(modelPath: string | undefined): Promise<GraphMod
} catch {
options.cacheSupported = false;
}
modelStats[shortModelName].cached = (options.cacheSupported && options.cacheModels) && Object.keys(cachedModels).includes(cachedModelName); // is model found in cache
modelStats[shortModelName].inCache = (options.cacheSupported && options.cacheModels) && Object.keys(cachedModels).includes(cachedModelName); // is model found in cache
const tfLoadOptions = typeof fetch === 'undefined' ? {} : { fetchFunc: (url, init?) => httpHandler(url, init) };
const model: GraphModel = new tf.GraphModel(modelStats[shortModelName].cached ? cachedModelName : modelUrl, tfLoadOptions) as unknown as GraphModel; // create model prototype and decide if load from cache or from original modelurl
const model: GraphModel = new tf.GraphModel(modelStats[shortModelName].inCache ? cachedModelName : modelUrl, tfLoadOptions) as unknown as GraphModel; // create model prototype and decide if load from cache or from original modelurl
let loaded = false;
try {
// @ts-ignore private function
@ -60,16 +63,16 @@ export async function loadModel(modelPath: string | undefined): Promise<GraphMod
if (options.debug) log('model load handler:', model['handler']);
// @ts-ignore private property
const artifacts = await model.handler.load(); // load manifest
modelStats[shortModelName].manifest = artifacts?.weightData?.byteLength || 0;
modelStats[shortModelName].sizeFromManifest = artifacts?.weightData?.byteLength || 0;
model.loadSync(artifacts); // load weights
// @ts-ignore private property
modelStats[shortModelName].weights = model?.artifacts?.weightData?.byteLength || 0;
if (options.verbose) log('load model:', model['modelUrl'], { bytes: modelStats[shortModelName].weights }, options);
modelStats[shortModelName].sizeLoadedWeights = model?.artifacts?.weightData?.byteLength || 0;
if (options.verbose) log('load model:', model['modelUrl'], { bytes: modelStats[shortModelName].sizeLoadedWeights }, options);
loaded = true;
} catch (err) {
log('error loading model:', modelUrl, err);
}
if (loaded && options.cacheModels && options.cacheSupported && !modelStats[shortModelName].cached) { // save model to cache
if (loaded && options.cacheModels && options.cacheSupported && !modelStats[shortModelName].inCache) { // save model to cache
try {
const saveResult = await model.save(cachedModelName);
log('model saved:', cachedModelName, saveResult);

48
test/build.log
View File

@ -1,24 +1,24 @@
2022-07-16 09:03:53 INFO:  Application: {"name":"@vladmandic/human","version":"2.8.1"}
2022-07-16 09:03:53 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-07-16 09:03:53 INFO:  Toolchain: {"build":"0.7.7","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.23.7","eslint":"8.19.0"}
2022-07-16 09:03:53 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-07-16 09:03:53 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-07-16 09:03:53 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608}
2022-07-16 09:03:53 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":73,"inputBytes":645512,"outputBytes":301528}
2022-07-16 09:03:53 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612}
2022-07-16 09:03:53 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":73,"inputBytes":645516,"outputBytes":301532}
2022-07-16 09:03:53 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664}
2022-07-16 09:03:53 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":73,"inputBytes":645568,"outputBytes":301582}
2022-07-16 09:03:53 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358}
2022-07-16 09:03:53 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583}
2022-07-16 09:03:53 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":73,"inputBytes":645487,"outputBytes":300416}
2022-07-16 09:03:54 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1140,"outputBytes":2799617}
2022-07-16 09:03:54 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":73,"inputBytes":3444521,"outputBytes":1676007}
2022-07-16 09:03:54 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":73,"inputBytes":3444521,"outputBytes":3073760}
2022-07-16 09:04:01 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":116}
2022-07-16 09:04:03 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":74,"generated":true}
2022-07-16 09:04:03 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6371,"outputBytes":3094}
2022-07-16 09:04:03 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-07-16 09:04:13 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0}
2022-07-16 09:04:13 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-07-16 09:04:13 INFO:  Done...
2022-07-17 21:29:42 INFO:  Application: {"name":"@vladmandic/human","version":"2.9.0"}
2022-07-17 21:29:42 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-07-17 21:29:42 INFO:  Toolchain: {"build":"0.7.7","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.23.8","eslint":"8.20.0"}
2022-07-17 21:29:42 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-07-17 21:29:42 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-07-17 21:29:42 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608}
2022-07-17 21:29:42 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":74,"inputBytes":647707,"outputBytes":303979}
2022-07-17 21:29:42 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612}
2022-07-17 21:29:42 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":74,"inputBytes":647711,"outputBytes":303983}
2022-07-17 21:29:42 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664}
2022-07-17 21:29:42 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":74,"inputBytes":647763,"outputBytes":304033}
2022-07-17 21:29:42 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358}
2022-07-17 21:29:42 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583}
2022-07-17 21:29:42 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":74,"inputBytes":647682,"outputBytes":302858}
2022-07-17 21:29:43 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1140,"outputBytes":2799617}
2022-07-17 21:29:43 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":74,"inputBytes":3446716,"outputBytes":1678559}
2022-07-17 21:29:43 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":74,"inputBytes":3446716,"outputBytes":3078304}
2022-07-17 21:29:49 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":30}
2022-07-17 21:29:52 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true}
2022-07-17 21:29:52 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6371,"outputBytes":3094}
2022-07-17 21:29:52 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-07-17 21:30:02 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0}
2022-07-17 21:30:02 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-07-17 21:30:02 INFO:  Done...