ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

prototype precompile pass

pull/356/head
Vladimir Mandic 2022-03-19 11:02:30 -04:00
parent 507d6fda02
commit 647953cb67
8 changed files with 760 additions and 723 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@
### **HEAD -> main** 2022/03/16 mandic00@live.com
- fix indexdb config check
### **2.6.4** 2022/02/27 mandic00@live.com

4
demo/typescript/index.js
View File

@ -4,8 +4,8 @@
author: <https://github.com/vladmandic>'
*/
import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+`
`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],v=o.getCapabilities?o.getCapabilities():"",g=o.getSettings?o.getSettings():"",u=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:g,constraints:u,capabilities:v}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y;
import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!0},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+`
`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],u=o.getCapabilities?o.getCapabilities():"",v=o.getSettings?o.getSettings():"",g=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:v,constraints:g,capabilities:u}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y;
/**
* Human demo for browsers
* @default Human Library

4
demo/typescript/index.js.map
View File

File diff suppressed because one or more lines are too long

2
demo/typescript/index.ts
View File

@ -17,7 +17,7 @@ const humanConfig: Partial<Config> = { // user configuration for human, used to
face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
body: { enabled: true },
hand: { enabled: true },
object: { enabled: false },
object: { enabled: true },
gesture: { enabled: true },
};

1
src/face/blazeface.ts
View File

@ -55,7 +55,6 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
// sanity check on input
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return [];
const t: Record<string, Tensor> = {};
t.resized = tf.image.resizeBilinear(inputImage, [inputSize, inputSize]);
t.div = tf.div(t.resized, constants.tf127);
t.normalized = tf.sub(t.div, constants.tf05);

55
src/warmup.ts
View File

@ -6,13 +6,13 @@ import { log, now, mergeDeep } from './util/util';
import * as sample from './sample';
import * as tf from '../dist/tfjs.esm.js';
import * as image from './image/image';
import { env } from './util/env';
import type { Config } from './config';
import type { Result } from './result';
import type { Human } from './human';
import type { Tensor } from './tfjs/types';
import { env } from './util/env';
import type { Human, Models } from './human';
import type { Tensor, GraphModel } from './tfjs/types';
async function warmupBitmap(instance: Human) {
async function warmupBitmap(instance: Human): Promise<Result | undefined> {
const b64toBlob = (base64: string, type = 'application/octet-stream') => fetch(`data:${type};base64,${base64}`).then((res) => res.blob());
let blob;
let res;
@ -99,6 +99,45 @@ async function warmupNode(instance: Human): Promise<Result | undefined> {
return res;
}
async function runInference(instance: Human) {
let res: Result | undefined;
if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
else res = await warmupNode(instance);
return res;
}
/** Runs pre-compile on all loaded models */
export async function runCompile(allModels: Models) {
const backendType = tf.getBackend();
const webGLBackend = tf.backend();
if ((backendType !== 'webgl' && backendType !== 'humangl') || (!webGLBackend || !webGLBackend.checkCompileCompletion)) {
log('compile pass: skip');
return;
}
const models = Object.values(allModels).filter((m) => m !== null) as GraphModel[];
tf.env().set('ENGINE_COMPILE_ONLY', true);
const numTensorsStart = tf.engine().state.numTensors;
for (const model of models) {
const shape = (model.inputs && model.inputs[0] && model.inputs[0].shape) ? [...model.inputs[0].shape] : [1, 64, 64, 3];
const dtype = (model.inputs && model.inputs[0] && model.inputs[0].dtype) ? model.inputs[0].dtype : 'float32';
for (let dim = 0; dim < shape.length; dim++) {
if (shape[dim] === -1) shape[dim] = dim === 0 ? 1 : 64; // override batch number and any dynamic dimensions
}
const tensor = tf.zeros(shape, dtype);
const res = await model.executeAsync(tensor);
if (Array.isArray(res)) res.forEach((t) => tf.dispose(t));
else tf.dispose(res);
tf.dispose(tensor);
}
const kernels = await webGLBackend.checkCompileCompletionAsync();
webGLBackend.getUniformLocations();
log('compile pass kernels:', kernels.length);
tf.env().set('ENGINE_COMPILE_ONLY', false);
const numTensorsEnd = tf.engine().state.numTensors;
if ((numTensorsEnd - numTensorsStart) > 0) log('tensor leak:', numTensorsEnd - numTensorsStart);
}
/** Warmup method pre-initializes all configured models for faster inference
* - can take significant time on startup
* - only used for `webgl` and `humangl` backends
@ -111,13 +150,11 @@ export async function warmup(instance: Human, userConfig?: Partial<Config>): Pro
if (!instance.config.warmup || instance.config.warmup.length === 0 || instance.config.warmup === 'none') {
return { face: [], body: [], hand: [], gesture: [], object: [], performance: instance.performance, timestamp: now(), persons: [], error: null };
}
let res;
return new Promise(async (resolve) => {
if (typeof createImageBitmap === 'function') res = await warmupBitmap(instance);
else if (typeof Image !== 'undefined' || env.Canvas !== undefined) res = await warmupCanvas(instance);
else res = await warmupNode(instance);
// await runCompile(instance.models);
const res = await runInference(instance);
const t1 = now();
if (instance.config.debug) log('Warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
if (instance.config.debug) log('warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');
instance.emit('warmup');
resolve(res);
});

48
test/build.log
View File

@ -1,24 +1,24 @@
2022-03-16 11:38:07 INFO:  Application: {"name":"@vladmandic/human","version":"2.6.4"}
2022-03-16 11:38:07 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-03-16 11:38:07 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.27","typescript":"4.6.2","typedoc":"0.22.13","eslint":"8.11.0"}
2022-03-16 11:38:07 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-03-16 11:38:07 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595}
2022-03-16 11:38:07 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":64,"inputBytes":562596,"outputBytes":292931}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599}
2022-03-16 11:38:07 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":64,"inputBytes":562600,"outputBytes":292935}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651}
2022-03-16 11:38:07 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":64,"inputBytes":562652,"outputBytes":292985}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1063,"outputBytes":394}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1068,"outputBytes":615}
2022-03-16 11:38:07 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":64,"inputBytes":562616,"outputBytes":291911}
2022-03-16 11:38:07 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1344096}
2022-03-16 11:38:08 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":64,"inputBytes":1906097,"outputBytes":1635010}
2022-03-16 11:38:08 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":64,"inputBytes":1906097,"outputBytes":2112905}
2022-03-16 11:38:13 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":112}
2022-03-16 11:38:15 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":69,"generated":true}
2022-03-16 11:38:15 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5863,"outputBytes":2915}
2022-03-16 11:38:15 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-03-16 11:38:22 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":95,"errors":0,"warnings":0}
2022-03-16 11:38:22 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-03-16 11:38:22 INFO:  Done...
2022-03-19 11:01:09 INFO:  Application: {"name":"@vladmandic/human","version":"2.6.4"}
2022-03-19 11:01:09 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-03-19 11:01:09 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.27","typescript":"4.6.2","typedoc":"0.22.13","eslint":"8.11.0"}
2022-03-19 11:01:09 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-03-19 11:01:09 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":64,"inputBytes":564348,"outputBytes":292966}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":64,"inputBytes":564352,"outputBytes":292970}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":64,"inputBytes":564404,"outputBytes":293020}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1063,"outputBytes":394}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1068,"outputBytes":615}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":64,"inputBytes":564368,"outputBytes":291946}
2022-03-19 11:01:09 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1344104}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":64,"inputBytes":1907857,"outputBytes":1635055}
2022-03-19 11:01:09 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":64,"inputBytes":1907857,"outputBytes":2113003}
2022-03-19 11:01:14 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":112}
2022-03-19 11:01:16 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":69,"generated":true}
2022-03-19 11:01:16 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5862,"outputBytes":2915}
2022-03-19 11:01:16 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-03-19 11:01:24 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":95,"errors":0,"warnings":0}
2022-03-19 11:01:24 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-03-19 11:01:24 INFO:  Done...

1368
test/test.log
View File

File diff suppressed because it is too large Load Diff