add basic anthropometry

CHANGELOG.md

@@ -9,8 +9,9 @@
   
 ## Changelog
   
-### **HEAD -> main** 2022/11/12 mandic00@live.com
+### **HEAD -> main** 2022/11/16 mandic00@live.com
 
+- include external typedefs
 - prepare external typedefs
 - rebuild all
 - include project files for types

TODO.md

@@ -69,6 +69,8 @@ Features:
   Image and video on-demand histogram equalization  
 - Support selecting specific video source when multiple cameras are present  
   See `human.webcam.enumerate()`  
+- Updated algorithm to determine distance from camera based on iris size  
+  See `human.result.face[n].iris`  
 
 Architecture:
 - Reduce build dependencies  

src/draw/face.ts

@@ -15,7 +15,7 @@ function drawLabels(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanv
   if (f.gender) l = replace(l, '[gender]', f.gender);
   if (f.genderScore) l = replace(l, '[genderScore]', 100 * f.genderScore);
   if (f.age) l = replace(l, '[age]', f.age);
-  if (f.iris) l = replace(l, '[distance]', f.iris);
+  if (f.iris) l = replace(l, '[distance]', 100 * f.iris);
   if (f.real) l = replace(l, '[real]', 100 * f.real);
   if (f.live) l = replace(l, '[live]', 100 * f.live);
   if (f.emotion && f.emotion.length > 0) {

src/face/anthropometry.ts

@@ -0,0 +1,28 @@
+import type { FaceResult } from '../result';
+
+export function calculateCameraDistance(face: FaceResult, width: number): number {
+  // iris points are [center, left, top, right, bottom]
+  // average size of human iris is 11.7mm - fairly constant for all ages/genders/races
+  const f = face?.annotations;
+  if (!f) return 0;
+  // get size of left and right iris in pixels, pick larger one as its likely to be more accurate and normalize to 0..1 range instead of pixels
+  const irisSize = Math.max(Math.abs(f.leftEyeIris[3][0] - f.leftEyeIris[1][0]), Math.abs(f.rightEyeIris[3][0] - f.rightEyeIris[1][0])) / width;
+  // distance of eye from camera in meters
+  const cameraDistance = Math.round(1.17 / irisSize) / 100;
+  return cameraDistance;
+}
+
+export function calculateEyesDistance(face: FaceResult, width: number): number {
+  // average distance between eyes is 65mm - fairly constant for typical adult male, but varies otherwise
+  const f = face?.annotations;
+  if (!f) return 0;
+  // get size of left and right iris in pixels, pick larger one as its likely to be more accurate and normalize to 0..1 range instead of pixels
+  const irisSize = Math.max(Math.abs(f.leftEyeIris[3][0] - f.leftEyeIris[1][0]), Math.abs(f.rightEyeIris[3][0] - f.rightEyeIris[1][0])) / width;
+  // pixel x and y distance of centers of left and right iris, you can use edges instead
+  const irisDistanceXY = [f.leftEyeIris[0][0] - f.rightEyeIris[0][0], f.leftEyeIris[0][1] - f.rightEyeIris[0][1]];
+  // absolute distance bewtween eyes in 0..1 range to account for head pitch (we can ignore yaw)
+  const irisDistance = Math.sqrt((irisDistanceXY[0] * irisDistanceXY[0]) + (irisDistanceXY[1] * irisDistanceXY[1])) / width;
+  // distance between eyes in meters
+  const eyesDistance = Math.round(1.17 * irisDistance / irisSize) / 100;
+  return eyesDistance;
+}

src/face/face.ts

@@ -21,6 +21,7 @@ import type { FaceResult, Emotion, Gender, Race } from '../result';
 import type { Tensor4D } from '../tfjs/types';
 import type { Human } from '../human';
 import { calculateFaceAngle } from './angles';
+import { calculateCameraDistance } from './anthropometry';
 
 interface DescRes { age: number, gender: Gender, genderScore: number, descriptor: number[], race?: { score: number, race: Race }[] }
 
@@ -40,7 +41,7 @@ export const detectFace = async (instance: Human /* instance of human */, input:
   const faceRes: FaceResult[] = [];
   instance.state = 'run:face';
 
-  const faces = await facemesh.predict(input, instance.config);
+  const faces: FaceResult[] = await facemesh.predict(input, instance.config);
   instance.performance.face = env.perfadd ? (instance.performance.face || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
   if (!input.shape || input.shape.length !== 4) return [];
   if (!faces) return [];
@@ -194,17 +195,7 @@ export const detectFace = async (instance: Human /* instance of human */, input:
       (descRes as DescRes).descriptor = insightfaceRes as number[];
     }
 
-    // calculate iris distance
-    // iris: array[ center, left, top, right, bottom]
-    if (!instance.config.face.iris?.enabled) {
-      // if (faces[i]?.annotations?.leftEyeIris) delete faces[i].annotations.leftEyeIris;
-      // if (faces[i]?.annotations?.rightEyeIris) delete faces[i].annotations.rightEyeIris;
-    }
-    const irisSize = (faces[i]?.annotations?.leftEyeIris?.[0] && faces[i]?.annotations?.rightEyeIris?.[0]
-      && (faces[i].annotations.leftEyeIris.length > 0) && (faces[i].annotations.rightEyeIris.length > 0)
-      && (faces[i].annotations.leftEyeIris[0] !== null) && (faces[i].annotations.rightEyeIris[0] !== null))
-      ? Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) / input.shape[2]
-      : 0; // note: average human iris size is 11.7mm
+    const irisSize = instance.config.face.iris?.enabled ? calculateCameraDistance(faces[i], input.shape[2]) : 0;
 
     // optionally return tensor
     const tensor = instance.config.face.detector?.return ? tf.squeeze(faces[i].tensor as Tensor4D) : null;
@@ -225,7 +216,7 @@ export const detectFace = async (instance: Human /* instance of human */, input:
     if (emotionRes) res.emotion = emotionRes as { score: number, emotion: Emotion }[];
     if (antispoofRes) res.real = antispoofRes as number;
     if (livenessRes) res.live = livenessRes as number;
-    if (irisSize && irisSize !== 0) res.iris = Math.trunc(500 / irisSize / 11.7) / 100;
+    if (irisSize > 0) res.iris = irisSize;
     if (rotation) res.rotation = rotation;
     if (tensor) res.tensor = tensor;
     faceRes.push(res);

test/build.log

@@ -1,50 +1,50 @@
-2022-11-16 11:18:02 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"} 
-2022-11-16 11:18:02 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"} 
-2022-11-16 11:18:02 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
-2022-11-16 11:18:02 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.14","typescript":"4.9.3","typedoc":"0.23.21","eslint":"8.27.0"} 
-2022-11-16 11:18:02 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
-2022-11-16 11:18:02 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":79,"inputBytes":672671,"outputBytes":318315} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":79,"inputBytes":672675,"outputBytes":318319} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":79,"inputBytes":673623,"outputBytes":318430} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":79,"inputBytes":672417,"outputBytes":316894} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144900} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":79,"inputBytes":1816647,"outputBytes":1458374} 
-2022-11-16 11:18:02 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":79,"inputBytes":1816647,"outputBytes":1917806} 
-2022-11-16 11:18:06 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} 
-2022-11-16 11:18:09 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true} 
-2022-11-16 11:18:09 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6136,"outputBytes":2914} 
-2022-11-16 11:18:09 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17166,"outputBytes":9243} 
-2022-11-16 11:18:17 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":113,"errors":0,"warnings":0} 
-2022-11-16 11:18:17 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
-2022-11-16 11:18:17 STATE: Copy: {"input":"node_modules/@vladmandic/tfjs/types/tfjs-core.d.ts","output":"types/tfjs-core.d.ts"} 
-2022-11-16 11:18:17 INFO:  Done... 
-2022-11-16 11:18:17 STATE: Copy: {"input":"node_modules/@vladmandic/tfjs/types/tfjs.d.ts","output":"types/tfjs.esm.d.ts"} 
-2022-11-16 11:18:17 STATE: Copy: {"input":"src/types/tsconfig.json","output":"types/tsconfig.json"} 
-2022-11-16 11:18:17 STATE: Copy: {"input":"src/types/eslint.json","output":"types/.eslintrc.json"} 
-2022-11-16 11:18:17 STATE: Copy: {"input":"src/types/tfjs.esm.d.ts","output":"dist/tfjs.esm.d.ts"} 
-2022-11-16 11:18:17 STATE: Filter: {"input":"types/tfjs-core.d.ts"} 
-2022-11-16 11:18:19 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195} 
-2022-11-16 11:18:19 STATE: Filter: {"input":"types/human.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.esm-nobundle.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.esm.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.node-gpu.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.node.d.ts"} 
-2022-11-16 11:18:19 STATE: Write: {"output":"dist/human.node-wasm.d.ts"} 
-2022-11-16 11:18:19 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} 
-2022-11-16 11:18:19 STATE: Models {"folder":"./models","models":12} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../human-models/models","models":43} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../blazepose/model/","models":4} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../anti-spoofing/model","models":1} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../efficientpose/models","models":3} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../insightface/models","models":5} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../movenet/models","models":3} 
-2022-11-16 11:18:19 STATE: Models {"folder":"../nanodet/models","models":4} 
-2022-11-16 11:18:19 STATE: Models: {"count":58,"totalSize":386543911} 
-2022-11-16 11:18:19 INFO:  Human Build complete... {"logFile":"test/build.log"} 
+2022-11-16 17:46:46 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"} 
+2022-11-16 17:46:46 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"} 
+2022-11-16 17:46:46 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 
+2022-11-16 17:46:46 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.14","typescript":"4.9.3","typedoc":"0.23.21","eslint":"8.27.0"} 
+2022-11-16 17:46:46 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 
+2022-11-16 17:46:46 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":80,"inputBytes":673738,"outputBytes":317961} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":80,"inputBytes":673742,"outputBytes":317965} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":80,"inputBytes":674690,"outputBytes":318076} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":80,"inputBytes":673484,"outputBytes":316540} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144900} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":80,"inputBytes":1817714,"outputBytes":1458000} 
+2022-11-16 17:46:46 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":80,"inputBytes":1817714,"outputBytes":1917498} 
+2022-11-16 17:46:50 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} 
+2022-11-16 17:46:52 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true} 
+2022-11-16 17:46:52 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6136,"outputBytes":2914} 
+2022-11-16 17:46:52 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17166,"outputBytes":9243} 
+2022-11-16 17:47:01 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":114,"errors":0,"warnings":0} 
+2022-11-16 17:47:01 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 
+2022-11-16 17:47:01 STATE: Copy: {"input":"node_modules/@vladmandic/tfjs/types/tfjs-core.d.ts","output":"types/tfjs-core.d.ts"} 
+2022-11-16 17:47:01 INFO:  Done... 
+2022-11-16 17:47:01 STATE: Copy: {"input":"node_modules/@vladmandic/tfjs/types/tfjs.d.ts","output":"types/tfjs.esm.d.ts"} 
+2022-11-16 17:47:01 STATE: Copy: {"input":"src/types/tsconfig.json","output":"types/tsconfig.json"} 
+2022-11-16 17:47:01 STATE: Copy: {"input":"src/types/eslint.json","output":"types/.eslintrc.json"} 
+2022-11-16 17:47:01 STATE: Copy: {"input":"src/types/tfjs.esm.d.ts","output":"dist/tfjs.esm.d.ts"} 
+2022-11-16 17:47:01 STATE: Filter: {"input":"types/tfjs-core.d.ts"} 
+2022-11-16 17:47:02 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195} 
+2022-11-16 17:47:02 STATE: Filter: {"input":"types/human.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.esm-nobundle.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.esm.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.node-gpu.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.node.d.ts"} 
+2022-11-16 17:47:02 STATE: Write: {"output":"dist/human.node-wasm.d.ts"} 
+2022-11-16 17:47:02 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} 
+2022-11-16 17:47:02 STATE: Models {"folder":"./models","models":12} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../human-models/models","models":43} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../blazepose/model/","models":4} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../anti-spoofing/model","models":1} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../efficientpose/models","models":3} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../insightface/models","models":5} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../movenet/models","models":3} 
+2022-11-16 17:47:02 STATE: Models {"folder":"../nanodet/models","models":4} 
+2022-11-16 17:47:02 STATE: Models: {"count":58,"totalSize":386543911} 
+2022-11-16 17:47:02 INFO:  Human Build complete... {"logFile":"test/build.log"} 

test/test-node-main.js

@@ -189,7 +189,7 @@ async function verifyDetails(human) {
   verify(res.face.length === 1, 'details face length', res.face.length);
   for (const face of res.face) {
     verify(face.score > 0.9 && face.boxScore > 0.9 && face.faceScore > 0.9, 'details face score', face.score, face.boxScore, face.faceScore);
-    verify(face.age > 23 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 105, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
+    verify(face.age > 23 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 0.5 && face.iris < 2.5, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
     verify(face.box.length === 4 && face.boxRaw.length === 4 && face.mesh.length === 478 && face.meshRaw.length === 478 && face.embedding.length === 1024, 'details face arrays', face.box.length, face.mesh.length, face.embedding.length);
     verify(face.emotion.length >= 2 && face.emotion[0].score > 0.30 && face.emotion[0].emotion === 'angry', 'details face emotion', face.emotion.length, face.emotion[0]);
     verify(face.real > 0.55, 'details face anti-spoofing', face.real);