placeholder for face contours

CHANGELOG.md

@@ -11,6 +11,7 @@
   
 ### **HEAD -> main** 2022/07/13 mandic00@live.com
 
+- add webview support
 - fix(gear): ensure gear.modelpath is used for loadmodel()
 - npm default install should be prod only
 - fix npm v7 compatibility

demo/index.js

@@ -56,10 +56,12 @@ let userConfig = {
   // body: { enabled: true, modelPath: 'movenet-multipose.json' },
   segmentation: { enabled: false },
   */
+  /*
   face: { iris: { enabled: false }, emotion: { enabled: false } },
   hand: { enabled: false },
   body: { enabled: false },
   gesture: { enabled: false },
+  */
 };
 
 const drawOptions = {

src/draw/face.ts

@@ -120,6 +120,13 @@ function drawFacePolygons(f: FaceResult, ctx: CanvasRenderingContext2D | Offscre
     }
     drawIrisElipse(f, ctx);
   }
+  /*
+  if (opt.drawPolygons && f.contours.length > 1) {
+    ctx.lineWidth = 5;
+    lines(ctx, f.contours, opt);
+  }
+  ctx.lineWidth = 1;
+  */
 }
 
 function drawFacePoints(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {

src/draw/primitives.ts

@@ -57,7 +57,7 @@ export function lines(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingCo
   ctx.beginPath();
   ctx.moveTo(points[0][0], points[0][1]);
   for (const pt of points) {
-    ctx.strokeStyle = colorDepth(pt[2], localOptions);
+    ctx.strokeStyle = colorDepth(pt[2] || 0, localOptions);
     ctx.lineTo(Math.trunc(pt[0]), Math.trunc(pt[1]));
   }
   ctx.stroke();

src/face/blazeface.ts

@@ -34,7 +34,7 @@ export async function load(config: Config): Promise<GraphModel> {
   return model;
 }
 
-function decodeBounds(boxOutputs: Tensor) {
+function decodeBoxes(boxOutputs: Tensor) {
   const t: Record<string, Tensor> = {};
   t.boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
   t.centers = tf.add(t.boxStarts, anchors);
@@ -71,7 +71,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
     t.batch = tf.squeeze(res);
   }
   tf.dispose(res);
-  t.boxes = decodeBounds(t.batch);
+  t.boxes = decodeBoxes(t.batch);
   t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
   t.sigmoid = tf.sigmoid(t.logits);
   t.scores = tf.squeeze(t.sigmoid);

src/face/facemesh.ts

@@ -59,6 +59,8 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
       score: 0,
       boxScore: 0,
       faceScore: 0,
+      // contoursRaw: [],
+      // contours: [],
       annotations: {} as Record<FaceLandmark, Point[]>,
     };
 
@@ -89,6 +91,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
       const confidenceT = results.find((t) => t.shape[t.shape.length - 1] === 1) as Tensor;
       const faceConfidence = await confidenceT.data();
       face.faceScore = Math.round(100 * faceConfidence[0]) / 100;
+
       if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh
         box.confidence = face.faceScore; // reset confidence of cached box
         if (config.face.mesh?.keepInvalid) {
@@ -99,7 +102,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
             ((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()),
             ((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()),
           ]);
-          face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]);
+          face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 1), pt[1] / (input.shape[1] || 1), (pt[2] || 0) / inputSize]);
           for (const key of Object.keys(coords.blazeFaceLandmarks)) {
             face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations
           }
@@ -121,6 +124,13 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
         const calculatedBox = { ...util.calculateFaceBox(face.mesh, box), confidence: box.confidence, landmarks: box.landmarks };
         face.box = util.clampBox(calculatedBox, input);
         face.boxRaw = util.getRawBox(calculatedBox, input);
+        /*
+        const contoursT = results.find((t) => t.shape[t.shape.length - 1] === 266) as Tensor;
+        const contoursData = contoursT && await contoursT.data(); // 133 x 2d points
+        face.contoursRaw = [];
+        for (let j = 0; j < contoursData.length / 2; j++) face.contoursRaw.push([contoursData[2 * j + 0] / inputSize, contoursData[2 * j + 1] / inputSize]);
+        face.contours = face.contoursRaw.map((c) => [Math.trunc((input.shape[2] || 1) * c[0]), Math.trunc((input.shape[1] || 1) * c[1])]);
+        */
         newCache.push(calculatedBox);
       }
       tf.dispose(results);

src/result.ts

@@ -40,7 +40,11 @@ export interface FaceResult {
   /** detected face mesh */
   mesh: Array<Point>
   /** detected face mesh normalized to 0..1 */
-  meshRaw: Array<Point>
+  meshRaw: Array<Point>,
+  /** face contours as array of 2d points normalized to 0..1 */
+  // contoursRaw: Array<[number, number]>,
+  /** face contours as array of 2d points */
+  // contours: Array<[number, number]>,
   /** mesh keypoints combined into annotated results */
   annotations: Record<FaceLandmark, Point[]>,
   /** detected age */

test/build.log

@@ -1,24 +1,24 @@
-2022-07-13 09:15:44 INFO:  Application: {"name":"@vladmandic/human","version":"2.8.1"} 
+2022-07-13 12:06:43 INFO:  Application: {"name":"@vladmandic/human","version":"2.8.1"} 
-2022-07-13 09:15:44 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
+2022-07-13 12:06:43 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
-2022-07-13 09:15:44 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.22.18","eslint":"8.19.0"} 
+2022-07-13 12:06:43 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.22.18","eslint":"8.19.0"} 
-2022-07-13 09:15:44 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
+2022-07-13 12:06:43 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
-2022-07-13 09:15:44 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
+2022-07-13 12:06:43 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 
-2022-07-13 09:15:44 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608} 
-2022-07-13 09:15:44 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":73,"inputBytes":644542,"outputBytes":301481} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":73,"inputBytes":645458,"outputBytes":301484} 
-2022-07-13 09:15:44 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":73,"inputBytes":644546,"outputBytes":301485} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":73,"inputBytes":645462,"outputBytes":301488} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":73,"inputBytes":644598,"outputBytes":301535} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":73,"inputBytes":645514,"outputBytes":301538} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358} 
+2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583} 
+2022-07-13 12:06:44 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":73,"inputBytes":644517,"outputBytes":300369} 
+2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":73,"inputBytes":645433,"outputBytes":300372} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1353528} 
+2022-07-13 12:06:44 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1364648} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":73,"inputBytes":1997462,"outputBytes":1652983} 
+2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":73,"inputBytes":2009498,"outputBytes":1664353} 
-2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":73,"inputBytes":1997462,"outputBytes":2140437} 
+2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":73,"inputBytes":2009498,"outputBytes":2154326} 
-2022-07-13 09:15:51 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":116} 
+2022-07-13 12:06:50 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":116} 
-2022-07-13 09:15:54 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":74,"generated":true} 
+2022-07-13 12:06:52 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":74,"generated":true} 
-2022-07-13 09:15:54 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6324,"outputBytes":3057} 
+2022-07-13 12:06:52 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6324,"outputBytes":3057} 
-2022-07-13 09:15:54 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
+2022-07-13 12:06:52 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} 
-2022-07-13 09:16:02 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0} 
+2022-07-13 12:07:01 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0} 
-2022-07-13 09:16:03 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
+2022-07-13 12:07:02 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
-2022-07-13 09:16:03 INFO:  Done... 
+2022-07-13 12:07:02 INFO:  Done...