enable precompile as part of warmup

CHANGELOG.md

@@ -11,9 +11,7 @@
   
 ### **HEAD -> main** 2022/04/14 mandic00@live.com
 
-
-### **origin/main** 2022/04/11 mandic00@live.com
-
+- change default face crop
 - face attention model is available in human-models
 - beta release 2.7
 - refactor draw methods

TODO.md

@@ -3,17 +3,19 @@
 ## Work in Progress
 
 Release 2.7:
-- add MediaPipe FaceMesh attention model  
-  model is available in @vladmandic/human-models repository  
+- Add **MediaPipe FaceMesh-with-Attention** model  
+  model is available in `@vladmandic/human-models` repository  
   to enable, set `config.face.attention = true`  
-  model replaces iris and eyes keypoints with high-detailed ones  
-  and adds additional keypoints for lips  
-- changed default face crop from 120% to 140%  
+  model replaces **iris**, **eyes** and **lips** keypoints with high-detailed ones  
+- Add model **pre-compile** phase to `warmup` method
+  result is speed-up to first inference by around ~30% for browser environments
+- Changed default face crop from 120% to 140%  
   to better utilize caching between frames  
-- refactor draw methods into separate modules  
-- add **ElectronJS** demo  
+- Refactor **draw** methods into separate modules  
+- Add **ElectronJS** demo  
   see <https://github.com/vladmandic/human-electron>  
-- update build platform and dependencies  
+- Update build platform and dependencies  
+- Update **TFJS**
 
 ### Exploring
 

demo/typescript/index.js

@@ -4,8 +4,8 @@
   author: <https://github.com/vladmandic>'
 */
 
-import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},cacheSensitivity:0,face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!0},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+`
-`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],v=o.getCapabilities?o.getCapabilities():"",u=o.getSettings?o.getSettings():"",g=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:u,constraints:g,capabilities:v}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y;
+import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+`
+`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],v=o.getCapabilities?o.getCapabilities():"",g=o.getSettings?o.getSettings():"",u=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:g,constraints:u,capabilities:v}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y;
 /**
  * Human demo for browsers
  * @default Human Library

demo/typescript/index.ts

@@ -14,11 +14,11 @@ const humanConfig: Partial<Config> = { // user configuration for human, used to
   // async: true,
   modelBasePath: '../../models',
   filter: { enabled: true, equalization: false },
-  cacheSensitivity: 0,
+  // cacheSensitivity: 0,
   face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
   body: { enabled: true },
   hand: { enabled: true },
-  object: { enabled: true },
+  object: { enabled: false },
   gesture: { enabled: true },
 };
 

package.json

@@ -53,7 +53,7 @@
     "tensorflow"
   ],
   "devDependencies": {
-    "@microsoft/api-extractor": "^7.22.1",
+    "@microsoft/api-extractor": "^7.22.2",
     "@tensorflow/tfjs": "^3.15.0",
     "@tensorflow/tfjs-backend-cpu": "^3.15.0",
     "@tensorflow/tfjs-backend-wasm": "^3.15.0",
@@ -73,7 +73,7 @@
     "@vladmandic/pilogger": "^0.4.4",
     "@vladmandic/tfjs": "github:vladmandic/tfjs",
     "canvas": "^2.9.1",
-    "dayjs": "^1.11.0",
+    "dayjs": "^1.11.1",
     "esbuild": "^0.14.36",
     "eslint": "8.13.0",
     "eslint-config-airbnb-base": "^15.0.0",

src/warmup.ts

@@ -10,7 +10,7 @@ import { env } from './util/env';
 import type { Config } from './config';
 import type { Result } from './result';
 import type { Human, Models } from './human';
-import type { Tensor, GraphModel } from './tfjs/types';
+import type { Tensor } from './tfjs/types';
 
 async function warmupBitmap(instance: Human): Promise<Result | undefined> {
   const b64toBlob = (base64: string, type = 'application/octet-stream') => fetch(`data:${type};base64,${base64}`).then((res) => res.blob());
@@ -112,26 +112,33 @@ export async function runCompile(allModels: Models) {
   const backendType = tf.getBackend();
   const webGLBackend = tf.backend();
   if ((backendType !== 'webgl' && backendType !== 'humangl') || (!webGLBackend || !webGLBackend.checkCompileCompletion)) {
-    log('compile pass: skip');
+    // log('compile pass: skip');
     return;
   }
-  const models = Object.values(allModels).filter((m) => m !== null) as GraphModel[];
   tf.env().set('ENGINE_COMPILE_ONLY', true);
   const numTensorsStart = tf.engine().state.numTensors;
-  for (const model of models) {
+  const compiledModels: string[] = [];
+  for (const [modelName, model] of Object.entries(allModels).filter(([key, val]) => (key !== null && val !== null))) {
     const shape = (model.inputs && model.inputs[0] && model.inputs[0].shape) ? [...model.inputs[0].shape] : [1, 64, 64, 3];
     const dtype = (model.inputs && model.inputs[0] && model.inputs[0].dtype) ? model.inputs[0].dtype : 'float32';
     for (let dim = 0; dim < shape.length; dim++) {
       if (shape[dim] === -1) shape[dim] = dim === 0 ? 1 : 64; // override batch number and any dynamic dimensions
     }
     const tensor = tf.zeros(shape, dtype);
-    const res = await model.executeAsync(tensor);
-    if (Array.isArray(res)) res.forEach((t) => tf.dispose(t));
-    else tf.dispose(res);
+    // const res = await model.executeAsync(tensor); // fails with current tfjs
+    try {
+      const res = model.execute(tensor);
+      compiledModels.push(modelName);
+      if (Array.isArray(res)) res.forEach((t) => tf.dispose(t));
+      else tf.dispose(res);
+    } catch {
+      log('compile fail model:', modelName);
+    }
     tf.dispose(tensor);
   }
   const kernels = await webGLBackend.checkCompileCompletionAsync();
   webGLBackend.getUniformLocations();
+  log('compile pass models:', compiledModels);
   log('compile pass kernels:', kernels.length);
   tf.env().set('ENGINE_COMPILE_ONLY', false);
   const numTensorsEnd = tf.engine().state.numTensors;
@@ -140,7 +147,7 @@ export async function runCompile(allModels: Models) {
 
 /** Warmup method pre-initializes all configured models for faster inference
  * - can take significant time on startup
- * - only used for `webgl` and `humangl` backends
+ * - only used in browser environments for `webgl` and `humangl` backends
  * @param userConfig?: Config
 */
 export async function warmup(instance: Human, userConfig?: Partial<Config>): Promise<Result | undefined> {
@@ -151,7 +158,7 @@ export async function warmup(instance: Human, userConfig?: Partial<Config>): Pro
     return { face: [], body: [], hand: [], gesture: [], object: [], performance: instance.performance, timestamp: now(), persons: [], error: null };
   }
   return new Promise(async (resolve) => {
-    // await runCompile(instance.models);
+    await runCompile(instance.models);
     const res = await runInference(instance);
     const t1 = now();
     if (instance.config.debug) log('warmup', instance.config.warmup, Math.round(t1 - t0), 'ms');

test/build.log

@@ -1,24 +1,24 @@
-2022-04-14 11:54:20 INFO:  Application: {"name":"@vladmandic/human","version":"2.7.0"} 
-2022-04-14 11:54:20 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
-2022-04-14 11:54:20 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.36","typescript":"4.6.3","typedoc":"0.22.15","eslint":"8.13.0"} 
-2022-04-14 11:54:20 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
-2022-04-14 11:54:20 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
-2022-04-14 11:54:20 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} 
-2022-04-14 11:54:20 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":72,"inputBytes":572717,"outputBytes":295399} 
-2022-04-14 11:54:20 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} 
-2022-04-14 11:54:20 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":72,"inputBytes":572721,"outputBytes":295403} 
-2022-04-14 11:54:20 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":72,"inputBytes":572773,"outputBytes":295453} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1063,"outputBytes":394} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1068,"outputBytes":615} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":72,"inputBytes":572737,"outputBytes":294360} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1349864} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":72,"inputBytes":1921986,"outputBytes":1643238} 
-2022-04-14 11:54:21 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":72,"inputBytes":1921986,"outputBytes":2123758} 
-2022-04-14 11:54:27 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":114} 
-2022-04-14 11:54:28 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":73,"generated":true} 
-2022-04-14 11:54:28 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5916,"outputBytes":2957} 
-2022-04-14 11:54:28 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
-2022-04-14 11:54:37 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":104,"errors":0,"warnings":0} 
-2022-04-14 11:54:37 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
-2022-04-14 11:54:37 INFO:  Done... 
+2022-04-15 07:53:06 INFO:  Application: {"name":"@vladmandic/human","version":"2.7.0"} 
+2022-04-15 07:53:06 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
+2022-04-15 07:53:06 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.36","typescript":"4.6.3","typedoc":"0.22.15","eslint":"8.13.0"} 
+2022-04-15 07:53:06 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
+2022-04-15 07:53:06 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":72,"inputBytes":573005,"outputBytes":296299} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":72,"inputBytes":573009,"outputBytes":296303} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":72,"inputBytes":573061,"outputBytes":296353} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":72,"inputBytes":572993,"outputBytes":295216} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1349864} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":72,"inputBytes":1922274,"outputBytes":1644071} 
+2022-04-15 07:53:06 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":72,"inputBytes":1922274,"outputBytes":2125347} 
+2022-04-15 07:53:11 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":114} 
+2022-04-15 07:53:13 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":73,"generated":true} 
+2022-04-15 07:53:13 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5920,"outputBytes":2938} 
+2022-04-15 07:53:13 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
+2022-04-15 07:53:21 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":104,"errors":0,"warnings":0} 
+2022-04-15 07:53:21 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
+2022-04-15 07:53:21 INFO:  Done... 

tfjs/tf-version.ts

@@ -4,7 +4,7 @@ import { version as tfjsCoreVersion } from '@tensorflow/tfjs-core/package.json';
 import { version as tfjsDataVersion } from '@tensorflow/tfjs-data/package.json';
 import { version as tfjsLayersVersion } from '@tensorflow/tfjs-layers/package.json';
 import { version as tfjsConverterVersion } from '@tensorflow/tfjs-converter/package.json';
-import { version as tfjsBackendCPUVersion } from '@tensorflow/tfjs-backend-cpu/package.json';
+// import { version as tfjsBackendCPUVersion } from '@tensorflow/tfjs-backend-cpu/package.json';
 import { version as tfjsBackendWebGLVersion } from '@tensorflow/tfjs-backend-webgl/package.json';
 import { version as tfjsBackendWASMVersion } from '@tensorflow/tfjs-backend-wasm/package.json';
 
@@ -14,7 +14,7 @@ export const version = {
   'tfjs-data': tfjsDataVersion,
   'tfjs-layers': tfjsLayersVersion,
   'tfjs-converter': tfjsConverterVersion,
-  'tfjs-backend-cpu': tfjsBackendCPUVersion,
+  // 'tfjs-backend-cpu': tfjsBackendCPUVersion,
   'tfjs-backend-webgl': tfjsBackendWebGLVersion,
   'tfjs-backend-wasm': tfjsBackendWASMVersion,
 };