breaking change: new similarity and match methods

CHANGELOG.md

@@ -9,13 +9,12 @@
   
 ## Changelog
   
-### **HEAD -> main** 2021/09/28 mandic00@live.com
+### **HEAD -> main** 2021/09/29 mandic00@live.com
 
+- release candidate
+- tweaked default values
 - enable handtrack as default model
 - redesign face processing
-
-### **origin/main** 2021/09/27 mandic00@live.com
-
 - refactoring
 - define app specific types
 - implement box caching for movenet

demo/facematch/facematch.js

@@ -75,11 +75,12 @@ async function SelectFaceCanvas(face) {
       ctx.font = 'small-caps 0.4rem "Lato"';
       ctx.fillStyle = 'rgba(255, 255, 255, 1)';
     }
-    const person = await human.match(face.embedding, db);
+    const arr = db.map((rec) => rec.embedding);
-    log('Match:', person);
+    const res = await human.match(face.embedding, arr);
+    log('Match:', db[res.index].name);
     document.getElementById('desc').innerHTML = `
       ${face.fileName}<br>
-      Match: ${Math.round(1000 * person.similarity) / 10}% ${person.name}
+      Match: ${Math.round(1000 * res.similarity) / 10}% ${db[res.index].name}
     `;
     embedding = face.embedding.map((a) => parseFloat(a.toFixed(4)));
     navigator.clipboard.writeText(`{"name":"unknown", "source":"${face.fileName}", "embedding":[${embedding}]},`);
@@ -91,7 +92,7 @@ async function SelectFaceCanvas(face) {
   for (const canvas of canvases) {
     // calculate similarity from selected face to current one in the loop
     const current = all[canvas.tag.sample][canvas.tag.face];
-    const similarity = human.similarity(face.embedding, current.embedding, 3);
+    const similarity = human.similarity(face.embedding, current.embedding);
     // get best match
     // draw the canvas
     canvas.title = similarity;
@@ -107,9 +108,10 @@ async function SelectFaceCanvas(face) {
     // identify person
     ctx.font = 'small-caps 1rem "Lato"';
     const start = performance.now();
-    const person = await human.match(current.embedding, db);
+    const arr = db.map((rec) => rec.embedding);
+    const res = await human.match(face.embedding, arr);
     time += (performance.now() - start);
-    if (person.similarity && person.similarity > minScore) ctx.fillText(`DB: ${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30);
+    if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30);
   }
 
   log('Analyzed:', 'Face:', canvases.length, 'DB:', db.length, 'Time:', time);
@@ -145,9 +147,10 @@ async function AddFaceCanvas(index, res, fileName) {
       ctx.font = 'small-caps 0.8rem "Lato"';
       ctx.fillStyle = 'rgba(255, 255, 255, 1)';
       ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
-      const person = await human.match(res.face[i].embedding, db);
+      const arr = db.map((rec) => rec.embedding);
+      const result = await human.match(res.face[i].embedding, arr);
       ctx.font = 'small-caps 1rem "Lato"';
-      if (person.similarity && person.similarity > minScore) ctx.fillText(`${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30);
+      if (result.similarity && res.similarity > minScore) ctx.fillText(`${(100 * result.similarity).toFixed(1)}% ${db[result.index].name}`, 4, canvas.height - 30);
     }
   }
   return ok;
@@ -212,7 +215,7 @@ async function main() {
     images = images.map((a) => `/human/samples/in/${a}`);
     log('Adding static image list:', images);
   } else {
-    log('Disoovered images:', images);
+    log('Discovered images:', images);
   }
 
   // download and analyze all images

src/face/faceres.ts

@@ -24,8 +24,6 @@ const last: Array<{
 let lastCount = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
 
-type DB = Array<{ name: string, source: string, embedding: number[] }>;
-
 export async function load(config: Config): Promise<GraphModel> {
   const modelUrl = join(config.modelBasePath, config.face.description?.modelPath || '');
   if (env.initial) model = null;
@@ -37,31 +35,6 @@ export async function load(config: Config): Promise<GraphModel> {
   return model;
 }
 
-export function similarity(embedding1: Array<number>, embedding2: Array<number>, order = 2): number {
-  if (!embedding1 || !embedding2) return 0;
-  if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
-  if (embedding1?.length !== embedding2?.length) return 0;
-  // general minkowski distance, euclidean distance is limited case where order is 2
-  const distance = 5.0 * embedding1
-    .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
-    .reduce((sum, now) => (sum + now), 0) // sum all distances
-    ** (1 / order); // get root of
-  const res = Math.max(0, 100 - distance) / 100.0;
-  return res;
-}
-
-export function match(embedding: Array<number>, db: DB, threshold = 0) {
-  let best = { similarity: 0, name: '', source: '', embedding: [] as number[] };
-  if (!embedding || !db || !Array.isArray(embedding) || !Array.isArray(db)) return best;
-  for (const f of db) {
-    if (f.embedding && f.name) {
-      const perc = similarity(embedding, f.embedding);
-      if (perc > threshold && perc > best.similarity) best = { ...f, similarity: perc };
-    }
-  }
-  return best;
-}
-
 export function enhance(input): Tensor {
   const image = tf.tidy(() => {
     // input received from detector is already normalized to 0..1

src/face/match.ts

@@ -0,0 +1,60 @@
+/** Defines Descriptor type */
+export type Descriptor = Array<number>
+
+/** Calculates distance between two descriptors
+ *  - Minkowski distance algorithm of nth order if `order` is different than 2
+ *  - Euclidean distance if `order` is 2 (default)
+ *
+ * Options:
+ * - `order`
+ *
+ * Note: No checks are performed for performance reasons so make sure to pass valid number arrays of equal length
+ */
+export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) {
+  // general minkowski distance, euclidean distance is limited case where order is 2
+  let sum = 0;
+  for (let i = 0; i < descriptor1.length; i++) {
+    const diff = (options.order === 2) ? (descriptor1[i] - descriptor2[i]) : (Math.abs(descriptor1[i] - descriptor2[i]));
+    sum += (options.order === 2) ? (diff * diff) : (diff ** options.order);
+  }
+  return sum;
+}
+
+/** Calculates normalized similarity between two descriptors based on their `distance`
+ */
+export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) {
+  const dist = distance(descriptor1, descriptor2, options);
+  const invert = (options.order === 2) ? Math.sqrt(dist) : dist ** (1 / options.order);
+  return Math.max(0, 100 - invert) / 100.0;
+}
+
+/** Matches given descriptor to a closest entry in array of descriptors
+ * @param descriptor face descriptor
+ * @param descriptors array of face descriptors to commpare given descriptor to
+ *
+ * Options:
+ * - `order` see {@link distance} method
+ * - `threshold` match will return result first result for which {@link distance} is below `threshold` even if there may be better results
+ *
+ * @returns object with index, distance and similarity
+ * - `index` index array index where best match was found or -1 if no matches
+ * - {@link distance} calculated `distance` of given descriptor to the best match
+ * - {@link similarity} calculated normalized `similarity` of given descriptor to the best match
+*/
+export function match(descriptor: Descriptor, descriptors: Array<Descriptor>, options = { order: 2, threshold: 0 }) {
+  if (!Array.isArray(descriptor) || !Array.isArray(descriptors) || descriptor.length < 64 || descriptors.length === 0 || descriptor.length !== descriptors[0].length) { // validate input
+    return { index: -1, distance: Number.POSITIVE_INFINITY, similarity: 0 };
+  }
+  let best = Number.MAX_SAFE_INTEGER;
+  let index = -1;
+  for (let i = 0; i < descriptors.length; i++) {
+    const res = distance(descriptor, descriptors[i], { order: options.order });
+    if (res < best) {
+      best = res;
+      index = i;
+    }
+    if (best < options.threshold) break;
+  }
+  best = (options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order);
+  return { index, distance: best, similarity: Math.max(0, 100 - best) / 100.0 };
+}

src/human.ts

@@ -2,49 +2,64 @@
  * Human main module
  */
 
+// module imports
 import { log, now, mergeDeep, validate } from './util/util';
-import { Config, defaults } from './config';
+import { defaults } from './config';
-import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
 import * as tf from '../dist/tfjs.esm.js';
-import * as models from './models';
+import * as app from '../package.json';
+import * as backend from './tfjs/backend';
+// import * as blazepose from './body/blazepose-v1';
+import * as blazepose from './body/blazepose';
+import * as centernet from './object/centernet';
+import * as draw from './util/draw';
+import * as efficientpose from './body/efficientpose';
+import * as env from './util/env';
 import * as face from './face/face';
 import * as facemesh from './face/facemesh';
 import * as faceres from './face/faceres';
-import * as posenet from './body/posenet';
+import * as gesture from './gesture/gesture';
-import * as handtrack from './hand/handtrack';
 import * as handpose from './handpose/handpose';
-// import * as blazepose from './body/blazepose-v1';
+import * as handtrack from './hand/handtrack';
-import * as blazepose from './body/blazepose';
+import * as humangl from './tfjs/humangl';
-import * as efficientpose from './body/efficientpose';
+import * as image from './image/image';
+import * as interpolate from './util/interpolate';
+import * as match from './face/match';
+import * as models from './models';
 import * as movenet from './body/movenet';
 import * as nanodet from './object/nanodet';
-import * as centernet from './object/centernet';
-import * as segmentation from './segmentation/segmentation';
-import * as gesture from './gesture/gesture';
-import * as image from './image/image';
-import * as draw from './util/draw';
 import * as persons from './util/persons';
-import * as interpolate from './util/interpolate';
+import * as posenet from './body/posenet';
-import * as env from './util/env';
+import * as segmentation from './segmentation/segmentation';
-import * as backend from './tfjs/backend';
-import * as humangl from './tfjs/humangl';
-import * as app from '../package.json';
 import * as warmups from './warmup';
+
+// type definitions
+import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
 import type { Tensor } from './tfjs/types';
 import type { DrawOptions } from './util/draw';
+import type { Input } from './image/image';
+import type { Config } from './config';
 
-// export types
+/** Defines configuration options used by all **Human** methods */
 export * from './config';
+
+/** Defines result types returned by all **Human** methods */
 export * from './result';
+
+/** Defines DrawOptions used by `human.draw.*` methods */
 export type { DrawOptions } from './util/draw';
 export { env, Env } from './util/env';
+
+/** Face descriptor type as number array */
+export type { Descriptor } from './face/match';
+
+/** Box and Point primitives */
 export { Box, Point } from './result';
+
+/** Defines all possible models used by **Human** library */
 export { Models } from './models';
 
-/** Defines all possible input types for **Human** detection
+/** Defines all possible input types for **Human** detection */
- * @typedef Input Type
+export { Input } from './image/image';
- */
-export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;
 
 /** Events dispatched by `human.events`
  *
@@ -139,7 +154,7 @@ export class Human {
    * - `warmup`: triggered when warmup is complete
    * - `error`: triggered on some errors
    */
-  events: EventTarget;
+  events: EventTarget | undefined;
   /** Reference face triangualtion array of 468 points, used for triangle references between points */
   faceTriangulation: typeof facemesh.triangulation;
   /** Refernce UV map of 468 values, used for 3D mapping of the face mesh */
@@ -176,7 +191,7 @@ export class Human {
     this.#analyzeMemoryLeaks = false;
     this.#checkSanity = false;
     this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 };
-    this.events = new EventTarget();
+    this.events = (typeof EventTarget !== 'undefined') ? new EventTarget() : undefined;
     // object that contains all initialized models
     this.models = new models.Models();
     // reexport draw methods
@@ -230,17 +245,22 @@ export class Human {
   }
 
   /** Reset configuration to default values */
-  reset() {
+  reset(): void {
     const currentBackend = this.config.backend; // save backend;
     this.config = JSON.parse(JSON.stringify(defaults));
     this.config.backend = currentBackend;
   }
 
   /** Validate current configuration schema */
-  validate(userConfig?: Partial<Config>) {
+  public validate(userConfig?: Partial<Config>) {
     return validate(defaults, userConfig || this.config);
   }
 
+  /** Exports face matching methods */
+  public similarity = match.similarity;
+  public distance = match.distance;
+  public match = match.match;
+
   /** Process input as return canvas and tensor
    *
    * @param input: {@link Input}
@@ -250,19 +270,6 @@ export class Human {
     return image.process(input, this.config);
   }
 
-  /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings)
-   * - Calculation is based on normalized Minkowski distance between two descriptors
-   * - Default is Euclidean distance which is Minkowski distance of 2nd order
-   *
-   * @param embedding1: face descriptor as array of numbers
-   * @param embedding2: face descriptor as array of numbers
-   * @returns similarity: number
-  */
-  // eslint-disable-next-line class-methods-use-this
-  similarity(embedding1: Array<number>, embedding2: Array<number>): number {
-    return faceres.similarity(embedding1, embedding2);
-  }
-
   /** Segmentation method takes any input and returns processed canvas with body segmentation
    *  - Optional parameter background is used to fill the background with specific input
    *  - Segmentation is not triggered as part of detect process
@@ -290,18 +297,6 @@ export class Human {
     return faceres.enhance(input);
   }
 
-  /** Math method find best match between provided face descriptor and predefined database of known descriptors
-   *
-   * @param faceEmbedding: face descriptor previsouly calculated on any face
-   * @param db: array of mapping of face descriptors to known values
-   * @param threshold: minimum score for matching to be considered in the result
-   * @returns best match
-   */
-  // eslint-disable-next-line class-methods-use-this
-  match(faceEmbedding: Array<number>, db: Array<{ name: string, source: string, embedding: number[] }>, threshold = 0): { name: string, source: string, similarity: number, embedding: number[] } {
-    return faceres.match(faceEmbedding, db, threshold);
-  }
-
   /** Explicit backend initialization
    *  - Normally done implicitly during initial load phase
    *  - Call to explictly register and initialize TFJS backend without any other operations
@@ -309,7 +304,7 @@ export class Human {
    *
    * @return Promise<void>
    */
-  async init() {
+  async init(): Promise<void> {
     await backend.check(this, true);
     await this.tf.ready();
     env.set(this.env);
@@ -321,7 +316,7 @@ export class Human {
    * @param userConfig?: {@link Config}
    * @return Promise<void>
   */
-  async load(userConfig?: Partial<Config>) {
+  async load(userConfig?: Partial<Config>): Promise<void> {
     this.state = 'load';
     const timeStamp = now();
     const count = Object.values(this.models).filter((model) => model).length;
@@ -354,7 +349,9 @@ export class Human {
 
   // emit event
   /** @hidden */
-  emit = (event: string) => this.events?.dispatchEvent(new Event(event));
+  emit = (event: string) => {
+    if (this.events && this.events.dispatchEvent) this.events?.dispatchEvent(new Event(event));
+  }
 
   /** Runs interpolation using last known result and returns smoothened result
    * Interpolation is based on time since last known result so can be called independently
@@ -362,7 +359,7 @@ export class Human {
    * @param result?: {@link Result} optional use specific result set to run interpolation on
    * @returns result: {@link Result}
    */
-  next(result: Result = this.result) {
+  next(result: Result = this.result): Result {
     return interpolate.calc(result) as Result;
   }
 

src/image/image.ts

@@ -9,7 +9,7 @@ import type { Config } from '../config';
 import { env } from '../util/env';
 import { log } from '../util/util';
 
-type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas;
+export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas;
 
 const maxSize = 2048;
 // internal temp canvases

src/models.ts

@@ -51,13 +51,13 @@ export class Models {
   segmentation: null | GraphModel | Promise<GraphModel> = null;
 }
 
-export function reset(instance: Human) {
+export function reset(instance: Human): void {
   // if (instance.config.debug) log('resetting loaded models');
   for (const model of Object.keys(instance.models)) instance.models[model] = null;
 }
 
 /** Load method preloads all instance.configured models on-demand */
-export async function load(instance: Human) {
+export async function load(instance: Human): Promise<void> {
   if (env.initial) reset(instance);
   if (instance.config.hand.enabled) { // handpose model is a combo that must be loaded as a whole
     if (!instance.models.handpose && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config);
@@ -87,7 +87,7 @@ export async function load(instance: Human) {
   }
 }
 
-export async function validate(instance) {
+export async function validate(instance: Human): Promise<void> {
   interface Op { name: string, category: string, op: string }
   const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div'];
   for (const defined of Object.keys(instance.models)) {

test/test-main.js

@@ -237,25 +237,23 @@ async function test(Human, inputConfig) {
   const desc3 = res3 && res3.face && res3.face[0] && res3.face[0].embedding ? [...res3.face[0].embedding] : null;
   if (!desc1 || !desc2 || !desc3 || desc1.length !== 1024 || desc2.length !== 1024 || desc3.length !== 1024) log('error', 'failed: face descriptor', desc1?.length, desc2?.length, desc3?.length);
   else log('state', 'passed: face descriptor');
-  res1 = Math.round(10 * human.similarity(desc1, desc2));
+  res1 = human.similarity(desc1, desc1);
-  res2 = Math.round(10 * human.similarity(desc1, desc3));
+  res2 = human.similarity(desc1, desc2);
-  res3 = Math.round(10 * human.similarity(desc2, desc3));
+  res3 = human.similarity(desc1, desc3);
-  if (res1 !== 5 || res2 !== 5 || res3 !== 5) log('error', 'failed: face similarity ', res1, res2, res3);
+  if (res1 < 1 || res2 < 0.9 || res3 < 0.85) log('error', 'failed: face similarity ', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
-  else log('state', 'passed: face similarity');
+  else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
 
   // test face matching
   log('info', 'test face matching');
-  let db = [];
+  const db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString());
-  try {
+  const arr = db.map((rec) => rec.embedding);
-    db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString());
+  if (db.length < 20) log('error', 'failed: face database ', db.length);
-  } catch { /***/ }
-  if (db.length < 100) log('error', 'failed: face database ', db.length);
   else log('state', 'passed: face database', db.length);
-  res1 = human.match(desc1, db);
+  res1 = human.match(desc1, arr);
-  res2 = human.match(desc2, db);
+  res2 = human.match(desc2, arr);
-  res3 = human.match(desc3, db);
+  res3 = human.match(desc3, arr);
-  if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1, res2, res3);
+  if (res1.index !== 4 || res2.index !== 4 || res3.index !== 4) log('error', 'failed: face match ', res1, res2, res3);
-  else log('state', 'passed: face match', { first: { name: res1.name, similarity: res1.similarity } }, { second: { name: res2.name, similarity: res2.similarity } }, { third: { name: res3.name, similarity: res3.similarity } });
+  else log('state', 'passed: face match', { first: { index: res1.index, similarity: res1.similarity } }, { second: { index: res2.index, similarity: res2.similarity } }, { third: { index: res3.index, similarity: res3.similarity } });
 
   // test object detection
   log('info', 'test object');