add face.mesh.keepInvalid config flag

demo/nodejs/node-simple.js

@@ -1,19 +1,27 @@
 const fs = require('fs');
+const process = require('process');
 
 // eslint-disable-next-line import/no-extraneous-dependencies, no-unused-vars, @typescript-eslint/no-unused-vars
 const tf = require('@tensorflow/tfjs-node'); // in nodejs environments tfjs-node is required to be loaded before human
 // const faceapi = require('@vladmandic/face-api'); // use this when human is installed as module (majority of use cases)
 const Human = require('../../dist/human.node.js'); // use this when using human in dev mode
 
-async function main(inputFile) {
-  const human = new Human.Human(); // create instance of human using default configuration
+const humanConfig = {
+  // add any custom config here
+};
+
+async function detect(inputFile) {
+  const human = new Human.Human(humanConfig); // create instance of human using default configuration
   await human.load(); // optional as models would be loaded on-demand first time they are required
   await human.warmup(); // optional as model warmup is performed on-demand first time its executed
   const buffer = fs.readFileSync(inputFile); // read file data into buffer
   const tensor = human.tf.node.decodeImage(buffer); // decode jpg data
+  // eslint-disable-next-line no-console
+  console.log('loaded input file:', inputFile, 'resolution:', tensor.shape);
   const result = await human.detect(tensor); // run detection; will initialize backend and on-demand load models
   // eslint-disable-next-line no-console
   console.log(result);
 }
 
-main('samples/in/ai-body.jpg');
+if (process.argv.length === 3) detect(process.argv[2]); // if input file is provided as cmdline parameter use it
+else detect('samples/in/ai-body.jpg'); // else use built-in test inputfile

package.json

@@ -53,19 +53,19 @@
     "tensorflow"
   ],
   "devDependencies": {
-    "@microsoft/api-extractor": "^7.24.0",
-    "@tensorflow/tfjs": "^3.17.0",
-    "@tensorflow/tfjs-backend-cpu": "^3.17.0",
-    "@tensorflow/tfjs-backend-wasm": "^3.17.0",
-    "@tensorflow/tfjs-backend-webgl": "^3.17.0",
-    "@tensorflow/tfjs-backend-webgpu": "0.0.1-alpha.10",
-    "@tensorflow/tfjs-converter": "^3.17.0",
-    "@tensorflow/tfjs-core": "^3.17.0",
-    "@tensorflow/tfjs-data": "^3.17.0",
-    "@tensorflow/tfjs-layers": "^3.17.0",
-    "@tensorflow/tfjs-node": "^3.17.0",
-    "@tensorflow/tfjs-node-gpu": "^3.17.0",
-    "@types/node": "^17.0.34",
+    "@microsoft/api-extractor": "^7.24.1",
+    "@tensorflow/tfjs": "^3.18.0",
+    "@tensorflow/tfjs-backend-cpu": "^3.18.0",
+    "@tensorflow/tfjs-backend-wasm": "^3.18.0",
+    "@tensorflow/tfjs-backend-webgl": "^3.18.0",
+    "@tensorflow/tfjs-backend-webgpu": "0.0.1-alpha.11",
+    "@tensorflow/tfjs-converter": "^3.18.0",
+    "@tensorflow/tfjs-core": "^3.18.0",
+    "@tensorflow/tfjs-data": "^3.18.0",
+    "@tensorflow/tfjs-layers": "^3.18.0",
+    "@tensorflow/tfjs-node": "^3.18.0",
+    "@tensorflow/tfjs-node-gpu": "^3.18.0",
+    "@types/node": "^17.0.35",
     "@types/offscreencanvas": "^2019.6.4",
     "@typescript-eslint/eslint-plugin": "^5.25.0",
     "@typescript-eslint/parser": "^5.25.0",
@@ -75,7 +75,7 @@
     "canvas": "^2.9.1",
     "dayjs": "^1.11.2",
     "esbuild": "^0.14.39",
-    "eslint": "8.15.0",
+    "eslint": "8.16.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-html": "^6.2.0",
     "eslint-plugin-import": "^2.26.0",

src/config.ts

@@ -35,7 +35,10 @@ export interface FaceDetectorConfig extends GenericConfig {
 }
 
 /** Mesh part of face configuration */
-export interface FaceMeshConfig extends GenericConfig {}
+export interface FaceMeshConfig extends GenericConfig {
+  /** Keep detected faces that cannot be verified using facemesh */
+  keepInvalid: boolean
+}
 
 /** Iris part of face configuration */
 export interface FaceIrisConfig extends GenericConfig {}
@@ -352,6 +355,7 @@ const config: Config = {
     mesh: {
       enabled: true,
       modelPath: 'facemesh.json',
+      keepInvalid: false,
     },
     attention: {
       enabled: false,

src/face/facemesh.ts

@@ -94,6 +94,19 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
       let rawCoords = await coordsReshaped.array();
       if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh
         box.confidence = face.faceScore; // reset confidence of cached box
+        if (config.face.mesh?.keepInvalid) {
+          face.box = util.clampBox(box, input);
+          face.boxRaw = util.getRawBox(box, input);
+          face.score = face.boxScore;
+          face.mesh = box.landmarks.map((pt) => [
+            ((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()),
+            ((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()),
+          ]);
+          face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]);
+          for (const key of Object.keys(coords.blazeFaceLandmarks)) {
+            face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations
+          }
+        }
       } else {
         if (config.face.attention?.enabled) {
           rawCoords = await attention.augment(rawCoords, results); // augment iris results using attention model results

src/face/facemeshutil.ts

@@ -121,7 +121,9 @@ export const rotatePoint = (homogeneousCoordinate, rotationMatrix) => [dot(homog
 export const xyDistanceBetweenPoints = (a, b) => Math.sqrt(((a[0] - b[0]) ** 2) + ((a[1] - b[1]) ** 2));
 
 export function generateAnchors(inputSize) {
-  const spec = { strides: [inputSize / 16, inputSize / 8], anchors: [2, 6] };
+  const spec = inputSize === 192
+    ? { strides: [4], anchors: [1] } // facemesh-detector
+    : { strides: [inputSize / 16, inputSize / 8], anchors: [2, 6] }; // blazeface
   const anchors: Array<[number, number]> = [];
   for (let i = 0; i < spec.strides.length; i++) {
     const stride = spec.strides[i];

src/tfjs/load.ts

@@ -22,7 +22,8 @@ export function setModelLoadOptions(config: Config) {
 }
 
 export async function loadModel(modelPath: string | undefined): Promise<GraphModel> {
-  const modelUrl = join(options.modelBasePath, modelPath || '');
+  let modelUrl = join(options.modelBasePath, modelPath || '');
+  if (!modelUrl.toLowerCase().endsWith('.json')) modelUrl += '.json';
   const modelPathSegments = modelUrl.split('/');
   const cachedModelName = 'indexeddb://' + modelPathSegments[modelPathSegments.length - 1].replace('.json', ''); // generate short model name for cache
   const cachedModels = await tf.io.listModels(); // list all models already in cache

test/build.log

@@ -1,24 +1,24 @@
-2022-05-18 17:41:21 INFO:  Application: {"name":"@vladmandic/human","version":"2.7.2"} 
-2022-05-18 17:41:21 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
-2022-05-18 17:41:21 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.39","typescript":"4.6.4","typedoc":"0.22.15","eslint":"8.15.0"} 
-2022-05-18 17:41:21 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
-2022-05-18 17:41:21 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":72,"inputBytes":606782,"outputBytes":297946} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":72,"inputBytes":606786,"outputBytes":297950} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":72,"inputBytes":606838,"outputBytes":298000} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":72,"inputBytes":606770,"outputBytes":296859} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1352584} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":72,"inputBytes":1958771,"outputBytes":1648490} 
-2022-05-18 17:41:21 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":72,"inputBytes":1958771,"outputBytes":2131466} 
-2022-05-18 17:41:26 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":114} 
-2022-05-18 17:41:28 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":73,"generated":true} 
-2022-05-18 17:41:28 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5967,"outputBytes":2980} 
-2022-05-18 17:41:28 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
-2022-05-18 17:41:36 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":104,"errors":0,"warnings":0} 
-2022-05-18 17:41:36 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
-2022-05-18 17:41:36 INFO:  Done... 
+2022-05-22 08:49:40 INFO:  Application: {"name":"@vladmandic/human","version":"2.7.2"} 
+2022-05-22 08:49:40 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
+2022-05-22 08:49:40 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.39","typescript":"4.6.4","typedoc":"0.22.15","eslint":"8.16.0"} 
+2022-05-22 08:49:40 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
+2022-05-22 08:49:40 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":72,"inputBytes":607902,"outputBytes":298472} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":72,"inputBytes":607906,"outputBytes":298476} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":72,"inputBytes":607958,"outputBytes":298526} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":72,"inputBytes":607890,"outputBytes":297382} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1352913} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":72,"inputBytes":1960220,"outputBytes":1649341} 
+2022-05-22 08:49:40 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":72,"inputBytes":1960220,"outputBytes":2132978} 
+2022-05-22 08:49:45 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":114} 
+2022-05-22 08:49:47 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":73,"generated":true} 
+2022-05-22 08:49:47 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5967,"outputBytes":2980} 
+2022-05-22 08:49:47 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
+2022-05-22 08:49:55 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":104,"errors":0,"warnings":0} 
+2022-05-22 08:49:56 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
+2022-05-22 08:49:56 INFO:  Done...