added face angle calculations

README.md

@@ -38,6 +38,7 @@ Unfortunately, changes ended up being too large for a simple pull request on ori
 - Added test/dev built-in HTTP & HTTPS Web server
 - Removed `mtcnn` and `tinyYolov2` models as they were non-functional in latest public version of `Face-API`  
   *If there is a demand, I can re-implement them back.*  
+- Added `face angle` calculations that returns `roll`, `yaw` and `pitch`  
 
 Which means valid models are **tinyFaceDetector** and **mobileNetv1**  
 
@@ -388,7 +389,7 @@ npm run build
 
 ## Face Mesh
 
-`FaceAPI` returns 68-point face mesh as detailed in the image below:
+`FaceAPI` landmark model returns 68-point face mesh as detailed in the image below:
 
 ![facemesh](example/facemesh.png)
 

dist/face-api.esm-nobundle.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1292,7 +1292,7 @@
       ]
     },
     "package.json": {
-      "bytes": 1878,
+      "bytes": 1914,
       "imports": []
     },
     "src/xception/extractParams.ts": {
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 313824
+      "bytes": 315464
     },
     "dist/face-api.esm-nobundle.js": {
       "imports": [],
@@ -2981,7 +2981,7 @@
           "bytesInOutput": 420
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 333
+          "bytesInOutput": 784
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1225
@@ -3164,7 +3164,7 @@
           "bytesInOutput": 751
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1342
+          "bytesInOutput": 1334
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 84
@@ -3188,7 +3188,7 @@
           "bytesInOutput": 443
         }
       },
-      "bytes": 82147
+      "bytes": 82590
     }
   }
 }

dist/face-api.esm.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 1464559
+      "bytes": 1466199
     },
     "dist/face-api.esm.js": {
       "imports": [],
@@ -2978,7 +2978,7 @@
           "bytesInOutput": 422
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 337
+          "bytesInOutput": 790
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1228
@@ -3164,7 +3164,7 @@
           "bytesInOutput": 1093
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1348
+          "bytesInOutput": 1340
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 87
@@ -3188,7 +3188,7 @@
           "bytesInOutput": 446
         }
       },
-      "bytes": 1126714
+      "bytes": 1127159
     }
   }
 }

dist/face-api.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 1464566
+      "bytes": 1466206
     },
     "dist/face-api.js": {
       "imports": [],
@@ -2860,7 +2860,7 @@
           "bytesInOutput": 422
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 337
+          "bytesInOutput": 790
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1228
@@ -3043,7 +3043,7 @@
           "bytesInOutput": 1093
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1348
+          "bytesInOutput": 1340
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 86
@@ -3067,7 +3067,7 @@
           "bytesInOutput": 446
         }
       },
-      "bytes": 1126877
+      "bytes": 1127322
     }
   }
 }

dist/face-api.node-cpu.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1292,7 +1292,7 @@
       ]
     },
     "package.json": {
-      "bytes": 1878,
+      "bytes": 1914,
       "imports": []
     },
     "src/xception/extractParams.ts": {
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 313700
+      "bytes": 315340
     },
     "dist/face-api.node-cpu.js": {
       "imports": [],
@@ -2860,7 +2860,7 @@
           "bytesInOutput": 420
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 333
+          "bytesInOutput": 784
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1225
@@ -3043,7 +3043,7 @@
           "bytesInOutput": 752
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1343
+          "bytesInOutput": 1335
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 84
@@ -3067,7 +3067,7 @@
           "bytesInOutput": 443
         }
       },
-      "bytes": 82859
+      "bytes": 83302
     }
   }
 }

dist/face-api.node-gpu.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1292,7 +1292,7 @@
       ]
     },
     "package.json": {
-      "bytes": 1878,
+      "bytes": 1914,
       "imports": []
     },
     "src/xception/extractParams.ts": {
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 313709
+      "bytes": 315349
     },
     "dist/face-api.node-gpu.js": {
       "imports": [],
@@ -2860,7 +2860,7 @@
           "bytesInOutput": 420
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 333
+          "bytesInOutput": 784
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1225
@@ -3043,7 +3043,7 @@
           "bytesInOutput": 752
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1343
+          "bytesInOutput": 1335
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 84
@@ -3067,7 +3067,7 @@
           "bytesInOutput": 443
         }
       },
-      "bytes": 82868
+      "bytes": 83311
     }
   }
 }

dist/face-api.node.json

@@ -1221,7 +1221,7 @@
       ]
     },
     "src/factories/WithFaceLandmarks.ts": {
-      "bytes": 1643,
+      "bytes": 3192,
       "imports": [
         {
           "path": "src/classes/FaceDetection.ts",
@@ -1292,7 +1292,7 @@
       ]
     },
     "package.json": {
-      "bytes": 1878,
+      "bytes": 1914,
       "imports": []
     },
     "src/xception/extractParams.ts": {
@@ -1830,7 +1830,7 @@
       "imports": []
     },
     "src/ssdMobilenetv1/SsdMobilenetv1.ts": {
-      "bytes": 3675,
+      "bytes": 3652,
       "imports": [
         {
           "path": "dist/tfjs.esm.js",
@@ -2322,7 +2322,7 @@
       ]
     },
     "src/globalApi/DetectFacesTasks.ts": {
-      "bytes": 4604,
+      "bytes": 4124,
       "imports": [
         {
           "path": "src/factories/WithFaceDetection.ts",
@@ -2363,7 +2363,7 @@
       ]
     },
     "src/globalApi/detectFaces.ts": {
-      "bytes": 638,
+      "bytes": 624,
       "imports": [
         {
           "path": "src/ssdMobilenetv1/SsdMobilenetv1Options.ts",
@@ -2591,7 +2591,7 @@
       "imports": [],
       "exports": [],
       "inputs": {},
-      "bytes": 313701
+      "bytes": 315341
     },
     "dist/face-api.node.js": {
       "imports": [],
@@ -2860,7 +2860,7 @@
           "bytesInOutput": 420
         },
         "src/factories/WithFaceLandmarks.ts": {
-          "bytesInOutput": 333
+          "bytesInOutput": 784
         },
         "src/draw/DrawFaceLandmarks.ts": {
           "bytesInOutput": 1225
@@ -3043,7 +3043,7 @@
           "bytesInOutput": 752
         },
         "src/globalApi/DetectFacesTasks.ts": {
-          "bytesInOutput": 1343
+          "bytesInOutput": 1335
         },
         "src/globalApi/detectFaces.ts": {
           "bytesInOutput": 84
@@ -3067,7 +3067,7 @@
           "bytesInOutput": 443
         }
       },
-      "bytes": 82860
+      "bytes": 83303
     }
   }
 }

example/webcam.js

@@ -28,7 +28,7 @@ function drawFaces(canvas, data, fps) {
   if (!ctx) return;
   ctx.clearRect(0, 0, canvas.width, canvas.height);
   // draw title
-  ctx.font = '1.4rem sans-serif';
+  ctx.font = '1.2rem sans-serif';
   ctx.fillStyle = 'white';
   ctx.fillText(`FPS: ${fps}`, 10, 25);
   for (const person of data) {
@@ -43,16 +43,18 @@ function drawFaces(canvas, data, fps) {
     ctx.globalAlpha = 1;
     // const expression = person.expressions.sort((a, b) => Object.values(a)[0] - Object.values(b)[0]);
     const expression = Object.entries(person.expressions).sort((a, b) => b[1] - a[1]);
-    ctx.fillText(`gender ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 45);
+    ctx.fillText(`gender ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 60);
-    ctx.fillText(`expression ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 25);
+    ctx.fillText(`expression ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 42);
-    ctx.fillText(`age ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 5);
+    ctx.fillText(`age ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 24);
+    ctx.fillText(`roll:${Math.trunc(1000 * person.angle.roll) / 1000} pitch:${Math.trunc(1000 * person.angle.pitch) / 1000} yaw:${Math.trunc(1000 * person.angle.yaw) / 1000}`, person.detection.box.x, person.detection.box.y - 6);
     // draw face points for each face
     ctx.fillStyle = 'lightblue';
     ctx.globalAlpha = 0.5;
     const pointSize = 2;
-    for (const pt of person.landmarks.positions) {
+    for (let i = 0; i < person.landmarks.positions.length; i++) {
       ctx.beginPath();
-      ctx.arc(pt.x, pt.y, pointSize, 0, 2 * Math.PI);
+      ctx.arc(person.landmarks.positions[i].x, person.landmarks.positions[i].y, pointSize, 0, 2 * Math.PI);
+      ctx.fillText(`${i}`, person.landmarks.positions[i].x + 4, person.landmarks.positions[i].y + 4);
       ctx.fill();
     }
   }

server/build.js

@@ -205,7 +205,6 @@ function compile(fileNames, options) {
 async function build(f, msg) {
   log.info('Build: file', msg, f, 'target:', common.target);
   if (!es) es = await esbuild.startService();
-  // common build options
   try {
     // rebuild all target groups and types
     for (const [targetGroupName, targetGroup] of Object.entries(targets)) {

server/serve.js

@@ -14,6 +14,7 @@ const http2 = require('http2');
 const path = require('path');
 // eslint-disable-next-line node/no-unpublished-require, import/no-extraneous-dependencies
 const chokidar = require('chokidar');
+// eslint-disable-next-line node/no-unpublished-require, import/no-extraneous-dependencies
 const log = require('@vladmandic/pilogger');
 const build = require('./build.js');
 

src/factories/WithFaceLandmarks.ts

@@ -21,6 +21,36 @@ export function isWithFaceLandmarks(obj: any): obj is WithFaceLandmarks<WithFace
     && obj['alignedRect'] instanceof FaceDetection;
 }
 
+function calculateFaceAngle(mesh) {
+  const radians = (a1, a2, b1, b2) => Math.atan2(b2 - a2, b1 - a1);
+
+  const angle = { roll: <number | undefined>undefined, pitch: <number | undefined>undefined, yaw: <number | undefined>undefined };
+
+  if (!mesh || !mesh._positions || mesh._positions.length !== 68) return angle;
+  const pt = mesh._positions;
+
+  // roll is face lean left/right
+  // comparing x,y of outside corners of leftEye and rightEye
+  angle.roll = radians(pt[36]._x, pt[36]._y, pt[45]._x, pt[45]._y);
+
+  // yaw is face turn left/right
+  // comparing x distance of bottom of nose to left and right edge of face
+  //       and y distance of top    of nose to left and right edge of face
+  // precision is lacking since coordinates are not precise enough
+  angle.pitch = radians(pt[30]._x - pt[0]._x, pt[27]._y - pt[0]._y, pt[16]._x - pt[30]._x, pt[27]._y - pt[16]._y);
+
+  // pitch is face move up/down
+  // comparing size of the box around the face with top and bottom of detected landmarks
+  // silly hack, but this gives us face compression on y-axis
+  // e.g., tilting head up hides the forehead that doesn't have any landmarks so ratio drops
+  // value is normalized to range, but is not in actual radians
+  const bottom = pt.reduce((prev, cur) => (prev < cur._y ? prev : cur._y), +Infinity);
+  const top = pt.reduce((prev, cur) => (prev > cur._y ? prev : cur._y), -Infinity);
+  angle.yaw = 10 * (mesh._imgDims._height / (top - bottom) / 1.45 - 1);
+
+  return angle;
+}
+
 export function extendWithFaceLandmarks<
   TSource extends WithFaceDetection<{}>,
   TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 >(sourceObj: TSource, unshiftedLandmarks: TFaceLandmarks): WithFaceLandmarks<TSource, TFaceLandmarks> {
@@ -30,11 +60,13 @@ export function extendWithFaceLandmarks<
   const rect = landmarks.align();
   const { imageDims } = sourceObj.detection;
   const alignedRect = new FaceDetection(sourceObj.detection.score, rect.rescale(imageDims.reverse()), imageDims);
+  const angle = calculateFaceAngle(unshiftedLandmarks);
 
   const extension = {
     landmarks,
     unshiftedLandmarks,
     alignedRect,
+    angle,
   };
 
   return { ...sourceObj, ...extension };

src/globalApi/DetectFacesTasks.ts

@@ -27,28 +27,13 @@ export class DetectAllFacesTask extends DetectFacesTaskBase<FaceDetection[]> {
   public async run(): Promise<FaceDetection[]> {
     const { input, options } = this;
 
-    // eslint-disable-next-line no-nested-ternary
+    let result;
-    const faceDetectionFunction = options instanceof TinyFaceDetectorOptions
+    if (options instanceof TinyFaceDetectorOptions) result = nets.tinyFaceDetector.locateFaces(input, options);
-      // eslint-disable-next-line no-shadow
+    else if (options instanceof SsdMobilenetv1Options) result = nets.ssdMobilenetv1.locateFaces(input, options);
-      ? (input: TNetInput) => nets.tinyFaceDetector.locateFaces(input, options)
+    else if (options instanceof TinyYolov2Options) result = nets.tinyYolov2.locateFaces(input, options);
-      : (
+    else throw new Error('detectFaces - expected options to be instance of TinyFaceDetectorOptions | SsdMobilenetv1Options | TinyYolov2Options');
-        // eslint-disable-next-line no-nested-ternary
-        options instanceof SsdMobilenetv1Options
-          // eslint-disable-next-line no-shadow
-          ? (input: TNetInput) => nets.ssdMobilenetv1.locateFaces(input, options)
-          : (
-            options instanceof TinyYolov2Options
-              // eslint-disable-next-line no-shadow
-              ? (input: TNetInput) => nets.tinyYolov2.locateFaces(input, options)
-              : null
-          )
-      );
 
-    if (!faceDetectionFunction) {
+    return result;
-      throw new Error('detectFaces - expected options to be instance of TinyFaceDetectorOptions | SsdMobilenetv1Options | MtcnnOptions | TinyYolov2Options');
-    }
-
-    return faceDetectionFunction(input);
   }
 
   private runAndExtendWithFaceDetections(): Promise<WithFaceDetection<{}>[]> {
@@ -87,9 +72,7 @@ export class DetectSingleFaceTask extends DetectFacesTaskBase<FaceDetection | un
     const faceDetections = await new DetectAllFacesTask(this.input, this.options);
     let faceDetectionWithHighestScore = faceDetections[0];
     faceDetections.forEach((faceDetection) => {
-      if (faceDetection.score > faceDetectionWithHighestScore.score) {
+      if (faceDetection.score > faceDetectionWithHighestScore.score) faceDetectionWithHighestScore = faceDetection;
-        faceDetectionWithHighestScore = faceDetection;
-      }
     });
     return faceDetectionWithHighestScore;
   }

src/globalApi/detectFaces.ts

@@ -3,16 +3,10 @@ import { SsdMobilenetv1Options } from '../ssdMobilenetv1/SsdMobilenetv1Options';
 import { DetectAllFacesTask, DetectSingleFaceTask } from './DetectFacesTasks';
 import { FaceDetectionOptions } from './types';
 
-export function detectSingleFace(
+export function detectSingleFace(input: TNetInput, options: FaceDetectionOptions = new SsdMobilenetv1Options()): DetectSingleFaceTask {
-  input: TNetInput,
-  options: FaceDetectionOptions = new SsdMobilenetv1Options(),
-): DetectSingleFaceTask {
   return new DetectSingleFaceTask(input, options);
 }
 
-export function detectAllFaces(
+export function detectAllFaces(input: TNetInput, options: FaceDetectionOptions = new SsdMobilenetv1Options()): DetectAllFacesTask {
-  input: TNetInput,
-  options: FaceDetectionOptions = new SsdMobilenetv1Options(),
-): DetectAllFacesTask {
   return new DetectAllFacesTask(input, options);
 }

src/ssdMobilenetv1/SsdMobilenetv1.ts

@@ -30,10 +30,7 @@ export class SsdMobilenetv1 extends NeuralNetwork<NetParams> {
       const x = tf.sub(tf.mul(batchTensor, tf.scalar(0.007843137718737125)), tf.scalar(1)) as tf.Tensor4D;
       const features = mobileNetV1(x, params.mobilenetv1);
 
-      const {
+      const { boxPredictions, classPredictions } = predictionLayer(features.out, features.conv11, params.prediction_layer);
-        boxPredictions,
-        classPredictions,
-      } = predictionLayer(features.out, features.conv11, params.prediction_layer);
 
       return outputLayer(boxPredictions, classPredictions, params.output_layer);
     });