add extra face rotation prior to mesh

CHANGELOG.md

@@ -9,8 +9,9 @@
   
 ## Changelog
   
-### **HEAD -> main** 2021/11/14 mandic00@live.com
+### **release 2.5.2** 2021/11/15 mandic00@live.com
 
+- improve error handling
 
 ### **2.5.2** 2021/11/14 mandic00@live.com
 

TODO.md

@@ -6,9 +6,9 @@
 
 ### Exploring
 
-- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
+- Optical flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
-- Histogram Equalization: Regular, Adaptive, Contrast Limited, CLAHE
+- Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE
-- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
+- TFLite models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
 - Body segmentation: `robust-video-matting`
 - TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
 
@@ -19,6 +19,7 @@
 #### WebGPU
 
 Experimental support only until support is officially added in Chromium
+
 - Performance issues:
   <https://github.com/tensorflow/tfjs/issues/5689>
 
@@ -39,7 +40,7 @@ MoveNet MultiPose model does not work with WASM backend due to missing F32 broad
 
 <br><hr><br>
 
-## Pending release notes:
+## Pending Release Notes
 
 New:
 - new demo `demos/faceid` that utilizes multiple algorithm  to validate input before triggering face recognition

demo/faceid/README.md

@@ -32,3 +32,10 @@ designed to serve as a quick check when used together with other indicators:
 - Checks if input has obvious artifacts due to recording (e.g. playing back phone recording of a face)
 - Configuration: `human.config.face.liveness`.enabled
 - Result: `human.result.face[0].live` as score
+
+### Models
+
+**FaceID** is compatible with
+- `faceres.json` (default) perfoms combined age/gender/descriptor analysis
+- `faceres-deep.json` higher resolution variation of `faceres`
+- `mobilefacenet` alternative model for face descriptor analysis

demo/facematch/facematch.js

@@ -19,10 +19,12 @@ const userConfig = {
   filter: {
     enabled: true,
     equalization: true,
+    width: 0,
   },
   face: {
     enabled: true,
-    detector: { rotation: true, return: true, maxDetected: 50 },
+    // detector: { rotation: false, return: true, maxDetected: 50, iouThreshold: 0.206, minConfidence: 0.122 },
+    detector: { return: true, rotation: true, maxDetected: 50, iouThreshold: 0.01, minConfidence: 0.2 },
     mesh: { enabled: true },
     iris: { enabled: false },
     emotion: { enabled: true },
@@ -138,7 +140,8 @@ async function SelectFaceCanvas(face) {
 async function AddFaceCanvas(index, res, fileName) {
   all[index] = res.face;
   for (const i in res.face) {
-    if (res.face[i].mesh.length === 0 || !res.face[i].tensor) continue; // did not get valid results
+    if (!res.face[i].tensor) continue; // did not get valid results
+    if ((res.face[i].faceScore || 0) < human.config.face.detector.minConfidence) continue; // face analysis score too low
     all[index][i].fileName = fileName;
     const canvas = document.createElement('canvas');
     canvas.tag = { sample: index, face: i, source: fileName };
@@ -177,9 +180,9 @@ async function AddImageElement(index, image, length) {
   return new Promise((resolve) => {
     const img = new Image(128, 128);
     img.onload = () => { // must wait until image is loaded
+      document.getElementById('images').appendChild(img); // and finally we can add it
       human.detect(img, userConfig).then((res) => {
         AddFaceCanvas(index, res, image); // then wait until image is analyzed
-        document.getElementById('images').appendChild(img); // and finally we can add it
         resolve(true);
       });
     };
@@ -236,12 +239,8 @@ async function main() {
     log('Discovered images:', images);
   }
 
-  // images = ['/samples/in/solvay1927.jpg'];
+  // images = ['/samples/in/person-lexi.jpg', '/samples/in/person-carolina.jpg', '/samples/in/solvay1927.jpg'];
 
-  // download and analyze all images
-  // const promises = [];
-  // for (let i = 0; i < images.length; i++) promises.push(AddImageElement(i, images[i], images.length));
-  // await Promise.all(promises);
   const t0 = human.now();
   for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length);
   const t1 = human.now();

demo/typescript/index.js

@@ -8,7 +8,7 @@
 import { Human } from "../../dist/human.esm.js";
 var humanConfig = {
   modelBasePath: "../../models",
-  filter: { equalization: false }
+  filter: { equalization: true }
 };
 var human = new Human(humanConfig);
 human.env["perfadd"] = false;

demo/typescript/index.ts

@@ -11,8 +11,8 @@ import { Human } from '../../dist/human.esm.js'; // equivalent of @vladmandic/Hu
 
 const humanConfig = { // user configuration for human, used to fine-tune behavior
   modelBasePath: '../../models',
-  filter: { equalization: false },
+  filter: { equalization: true },
-  // backend: 'webgpu',
+  // backend: 'webgpu' as 'webgpu,
   // async: true,
   // face: { enabled: false, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
   // body: { enabled: false },

src/face/blazeface.ts

@@ -13,7 +13,6 @@ import type { Point } from '../result';
 
 const keypointsCount = 6;
 let model: GraphModel | null;
-let anchorsData: [number, number][] = [];
 let anchors: Tensor | null = null;
 let inputSize = 0;
 
@@ -27,9 +26,7 @@ export async function load(config: Config): Promise<GraphModel> {
     else if (config.debug) log('load model:', model['modelUrl']);
   } else if (config.debug) log('cached model:', model['modelUrl']);
   inputSize = model.inputs[0].shape ? model.inputs[0].shape[2] : 0;
-  if (inputSize === -1) inputSize = 64;
+  anchors = tf.tensor2d(util.generateAnchors(inputSize));
-  anchorsData = util.generateAnchors(inputSize);
-  anchors = tf.tensor2d(anchorsData);
   return model;
 }
 
@@ -73,7 +70,6 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
   t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
   t.sigmoid = tf.sigmoid(t.logits);
   t.scores = tf.squeeze(t.sigmoid);
-
   t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
   const nms = await t.nms.array() as number[];
   const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = [];
@@ -86,12 +82,11 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
       b.slice = tf.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
       b.squeeze = tf.squeeze(b.slice);
       b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
-      b.startPoint = tf.slice(b.bbox, [0, 0], [-1, 2]);
+      const points = await b.bbox.data();
-      b.endPoint = tf.slice(b.bbox, [0, 2], [-1, 2]);
       boxes.push({
         box: {
-          startPoint: (await b.startPoint.data()) as unknown as Point,
+          startPoint: [points[0], points[1]] as Point,
-          endPoint: (await b.endPoint.data()) as unknown as Point,
+          endPoint: [points[2], points[3]] as Point,
         },
         landmarks: (await b.landmarks.array()) as Point[],
         confidence,

src/face/facemesh.ts

@@ -41,7 +41,10 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
         landmarks: possible.landmarks,
         confidence: possible.confidence,
       };
-      boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(config.face.detector?.cropFactor || 1.6))));
+      const boxScaled = util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor);
+      const boxEnlarged = util.enlargeBox(boxScaled, Math.sqrt(config.face.detector?.cropFactor || 1.6));
+      const boxSquared = util.squarifyBox(boxEnlarged);
+      boxCache.push(boxSquared);
     }
     skipped = 0;
   } else {
@@ -67,7 +70,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
     };
 
     // optional rotation correction based on detector data only if mesh is disabled otherwise perform it later when we have more accurate mesh data. if no rotation correction this function performs crop
-    [angle, rotationMatrix, face.tensor] = util.correctFaceRotation(!config.face.mesh?.enabled && config.face.detector?.rotation, box, input, config.face.mesh?.enabled ? inputSize : blazeface.size());
+    [angle, rotationMatrix, face.tensor] = util.correctFaceRotation(config.face.detector?.rotation, box, input, config.face.mesh?.enabled ? inputSize : blazeface.size());
     if (config?.filter?.equalization) {
       const equilized = await histogramEqualization(face.tensor as Tensor);
       tf.dispose(face.tensor);

src/face/facemeshutil.ts

@@ -31,8 +31,8 @@ export const getRawBox = (box, input): Box => (box ? [
 ] : [0, 0, 0, 0]);
 
 export const scaleBoxCoordinates = (box, factor) => {
-  const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
+  const startPoint: Point = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
-  const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
+  const endPoint: Point = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
   return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence };
 };
 

src/image/image.ts

@@ -262,7 +262,7 @@ const checksum = async (input: Tensor): Promise<number> => { // use tf sum or js
 
 export async function skip(config, input: Tensor) {
   let skipFrame = false;
-  if (config.cacheSensitivity === 0) return skipFrame;
+  if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis
 
   /*
   const checkSum = await checksum(input);