diff --git a/CHANGELOG.md b/CHANGELOG.md index 84eb5bea..edada879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,12 @@ ## Changelog -### **HEAD -> main** 2021/09/28 mandic00@live.com +### **HEAD -> main** 2021/09/29 mandic00@live.com +- release candidate +- tweaked default values - enable handtrack as default model - redesign face processing - -### **origin/main** 2021/09/27 mandic00@live.com - - refactoring - define app specific types - implement box caching for movenet diff --git a/demo/facematch/facematch.js b/demo/facematch/facematch.js index b1b02678..fa0eab32 100644 --- a/demo/facematch/facematch.js +++ b/demo/facematch/facematch.js @@ -75,11 +75,12 @@ async function SelectFaceCanvas(face) { ctx.font = 'small-caps 0.4rem "Lato"'; ctx.fillStyle = 'rgba(255, 255, 255, 1)'; } - const person = await human.match(face.embedding, db); - log('Match:', person); + const arr = db.map((rec) => rec.embedding); + const res = await human.match(face.embedding, arr); + log('Match:', db[res.index].name); document.getElementById('desc').innerHTML = ` ${face.fileName}
- Match: ${Math.round(1000 * person.similarity) / 10}% ${person.name} + Match: ${Math.round(1000 * res.similarity) / 10}% ${db[res.index].name} `; embedding = face.embedding.map((a) => parseFloat(a.toFixed(4))); navigator.clipboard.writeText(`{"name":"unknown", "source":"${face.fileName}", "embedding":[${embedding}]},`); @@ -91,7 +92,7 @@ async function SelectFaceCanvas(face) { for (const canvas of canvases) { // calculate similarity from selected face to current one in the loop const current = all[canvas.tag.sample][canvas.tag.face]; - const similarity = human.similarity(face.embedding, current.embedding, 3); + const similarity = human.similarity(face.embedding, current.embedding); // get best match // draw the canvas canvas.title = similarity; @@ -107,9 +108,10 @@ async function SelectFaceCanvas(face) { // identify person ctx.font = 'small-caps 1rem "Lato"'; const start = performance.now(); - const person = await human.match(current.embedding, db); + const arr = db.map((rec) => rec.embedding); + const res = await human.match(face.embedding, arr); time += (performance.now() - start); - if (person.similarity && person.similarity > minScore) ctx.fillText(`DB: ${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30); + if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30); } log('Analyzed:', 'Face:', canvases.length, 'DB:', db.length, 'Time:', time); @@ -145,9 +147,10 @@ async function AddFaceCanvas(index, res, fileName) { ctx.font = 'small-caps 0.8rem "Lato"'; ctx.fillStyle = 'rgba(255, 255, 255, 1)'; ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6); - const person = await human.match(res.face[i].embedding, db); + const arr = db.map((rec) => rec.embedding); + const result = await human.match(res.face[i].embedding, arr); ctx.font = 'small-caps 1rem "Lato"'; - if (person.similarity && person.similarity > minScore) ctx.fillText(`${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30); + if (result.similarity && res.similarity > minScore) ctx.fillText(`${(100 * result.similarity).toFixed(1)}% ${db[result.index].name}`, 4, canvas.height - 30); } } return ok; @@ -212,7 +215,7 @@ async function main() { images = images.map((a) => `/human/samples/in/${a}`); log('Adding static image list:', images); } else { - log('Disoovered images:', images); + log('Discovered images:', images); } // download and analyze all images diff --git a/src/face/faceres.ts b/src/face/faceres.ts index 7543a35f..cfa89e18 100644 --- a/src/face/faceres.ts +++ b/src/face/faceres.ts @@ -24,8 +24,6 @@ const last: Array<{ let lastCount = 0; let skipped = Number.MAX_SAFE_INTEGER; -type DB = Array<{ name: string, source: string, embedding: number[] }>; - export async function load(config: Config): Promise { const modelUrl = join(config.modelBasePath, config.face.description?.modelPath || ''); if (env.initial) model = null; @@ -37,31 +35,6 @@ export async function load(config: Config): Promise { return model; } -export function similarity(embedding1: Array, embedding2: Array, order = 2): number { - if (!embedding1 || !embedding2) return 0; - if (embedding1?.length === 0 || embedding2?.length === 0) return 0; - if (embedding1?.length !== embedding2?.length) return 0; - // general minkowski distance, euclidean distance is limited case where order is 2 - const distance = 5.0 * embedding1 - .map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared - .reduce((sum, now) => (sum + now), 0) // sum all distances - ** (1 / order); // get root of - const res = Math.max(0, 100 - distance) / 100.0; - return res; -} - -export function match(embedding: Array, db: DB, threshold = 0) { - let best = { similarity: 0, name: '', source: '', embedding: [] as number[] }; - if (!embedding || !db || !Array.isArray(embedding) || !Array.isArray(db)) return best; - for (const f of db) { - if (f.embedding && f.name) { - const perc = similarity(embedding, f.embedding); - if (perc > threshold && perc > best.similarity) best = { ...f, similarity: perc }; - } - } - return best; -} - export function enhance(input): Tensor { const image = tf.tidy(() => { // input received from detector is already normalized to 0..1 diff --git a/src/face/match.ts b/src/face/match.ts new file mode 100644 index 00000000..d0ee82f3 --- /dev/null +++ b/src/face/match.ts @@ -0,0 +1,60 @@ +/** Defines Descriptor type */ +export type Descriptor = Array + +/** Calculates distance between two descriptors + * - Minkowski distance algorithm of nth order if `order` is different than 2 + * - Euclidean distance if `order` is 2 (default) + * + * Options: + * - `order` + * + * Note: No checks are performed for performance reasons so make sure to pass valid number arrays of equal length + */ +export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) { + // general minkowski distance, euclidean distance is limited case where order is 2 + let sum = 0; + for (let i = 0; i < descriptor1.length; i++) { + const diff = (options.order === 2) ? (descriptor1[i] - descriptor2[i]) : (Math.abs(descriptor1[i] - descriptor2[i])); + sum += (options.order === 2) ? (diff * diff) : (diff ** options.order); + } + return sum; +} + +/** Calculates normalized similarity between two descriptors based on their `distance` + */ +export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) { + const dist = distance(descriptor1, descriptor2, options); + const invert = (options.order === 2) ? Math.sqrt(dist) : dist ** (1 / options.order); + return Math.max(0, 100 - invert) / 100.0; +} + +/** Matches given descriptor to a closest entry in array of descriptors + * @param descriptor face descriptor + * @param descriptors array of face descriptors to commpare given descriptor to + * + * Options: + * - `order` see {@link distance} method + * - `threshold` match will return result first result for which {@link distance} is below `threshold` even if there may be better results + * + * @returns object with index, distance and similarity + * - `index` index array index where best match was found or -1 if no matches + * - {@link distance} calculated `distance` of given descriptor to the best match + * - {@link similarity} calculated normalized `similarity` of given descriptor to the best match +*/ +export function match(descriptor: Descriptor, descriptors: Array, options = { order: 2, threshold: 0 }) { + if (!Array.isArray(descriptor) || !Array.isArray(descriptors) || descriptor.length < 64 || descriptors.length === 0 || descriptor.length !== descriptors[0].length) { // validate input + return { index: -1, distance: Number.POSITIVE_INFINITY, similarity: 0 }; + } + let best = Number.MAX_SAFE_INTEGER; + let index = -1; + for (let i = 0; i < descriptors.length; i++) { + const res = distance(descriptor, descriptors[i], { order: options.order }); + if (res < best) { + best = res; + index = i; + } + if (best < options.threshold) break; + } + best = (options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order); + return { index, distance: best, similarity: Math.max(0, 100 - best) / 100.0 }; +} diff --git a/src/human.ts b/src/human.ts index a807c25c..8789c014 100644 --- a/src/human.ts +++ b/src/human.ts @@ -2,49 +2,64 @@ * Human main module */ +// module imports import { log, now, mergeDeep, validate } from './util/util'; -import { Config, defaults } from './config'; -import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result'; +import { defaults } from './config'; import * as tf from '../dist/tfjs.esm.js'; -import * as models from './models'; +import * as app from '../package.json'; +import * as backend from './tfjs/backend'; +// import * as blazepose from './body/blazepose-v1'; +import * as blazepose from './body/blazepose'; +import * as centernet from './object/centernet'; +import * as draw from './util/draw'; +import * as efficientpose from './body/efficientpose'; +import * as env from './util/env'; import * as face from './face/face'; import * as facemesh from './face/facemesh'; import * as faceres from './face/faceres'; -import * as posenet from './body/posenet'; -import * as handtrack from './hand/handtrack'; +import * as gesture from './gesture/gesture'; import * as handpose from './handpose/handpose'; -// import * as blazepose from './body/blazepose-v1'; -import * as blazepose from './body/blazepose'; -import * as efficientpose from './body/efficientpose'; +import * as handtrack from './hand/handtrack'; +import * as humangl from './tfjs/humangl'; +import * as image from './image/image'; +import * as interpolate from './util/interpolate'; +import * as match from './face/match'; +import * as models from './models'; import * as movenet from './body/movenet'; import * as nanodet from './object/nanodet'; -import * as centernet from './object/centernet'; -import * as segmentation from './segmentation/segmentation'; -import * as gesture from './gesture/gesture'; -import * as image from './image/image'; -import * as draw from './util/draw'; import * as persons from './util/persons'; -import * as interpolate from './util/interpolate'; -import * as env from './util/env'; -import * as backend from './tfjs/backend'; -import * as humangl from './tfjs/humangl'; -import * as app from '../package.json'; +import * as posenet from './body/posenet'; +import * as segmentation from './segmentation/segmentation'; import * as warmups from './warmup'; + +// type definitions +import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result'; import type { Tensor } from './tfjs/types'; import type { DrawOptions } from './util/draw'; +import type { Input } from './image/image'; +import type { Config } from './config'; -// export types +/** Defines configuration options used by all **Human** methods */ export * from './config'; + +/** Defines result types returned by all **Human** methods */ export * from './result'; + +/** Defines DrawOptions used by `human.draw.*` methods */ export type { DrawOptions } from './util/draw'; export { env, Env } from './util/env'; + +/** Face descriptor type as number array */ +export type { Descriptor } from './face/match'; + +/** Box and Point primitives */ export { Box, Point } from './result'; + +/** Defines all possible models used by **Human** library */ export { Models } from './models'; -/** Defines all possible input types for **Human** detection - * @typedef Input Type - */ -export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; +/** Defines all possible input types for **Human** detection */ +export { Input } from './image/image'; /** Events dispatched by `human.events` * @@ -139,7 +154,7 @@ export class Human { * - `warmup`: triggered when warmup is complete * - `error`: triggered on some errors */ - events: EventTarget; + events: EventTarget | undefined; /** Reference face triangualtion array of 468 points, used for triangle references between points */ faceTriangulation: typeof facemesh.triangulation; /** Refernce UV map of 468 values, used for 3D mapping of the face mesh */ @@ -176,7 +191,7 @@ export class Human { this.#analyzeMemoryLeaks = false; this.#checkSanity = false; this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 }; - this.events = new EventTarget(); + this.events = (typeof EventTarget !== 'undefined') ? new EventTarget() : undefined; // object that contains all initialized models this.models = new models.Models(); // reexport draw methods @@ -230,17 +245,22 @@ export class Human { } /** Reset configuration to default values */ - reset() { + reset(): void { const currentBackend = this.config.backend; // save backend; this.config = JSON.parse(JSON.stringify(defaults)); this.config.backend = currentBackend; } /** Validate current configuration schema */ - validate(userConfig?: Partial) { + public validate(userConfig?: Partial) { return validate(defaults, userConfig || this.config); } + /** Exports face matching methods */ + public similarity = match.similarity; + public distance = match.distance; + public match = match.match; + /** Process input as return canvas and tensor * * @param input: {@link Input} @@ -250,19 +270,6 @@ export class Human { return image.process(input, this.config); } - /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings) - * - Calculation is based on normalized Minkowski distance between two descriptors - * - Default is Euclidean distance which is Minkowski distance of 2nd order - * - * @param embedding1: face descriptor as array of numbers - * @param embedding2: face descriptor as array of numbers - * @returns similarity: number - */ - // eslint-disable-next-line class-methods-use-this - similarity(embedding1: Array, embedding2: Array): number { - return faceres.similarity(embedding1, embedding2); - } - /** Segmentation method takes any input and returns processed canvas with body segmentation * - Optional parameter background is used to fill the background with specific input * - Segmentation is not triggered as part of detect process @@ -290,18 +297,6 @@ export class Human { return faceres.enhance(input); } - /** Math method find best match between provided face descriptor and predefined database of known descriptors - * - * @param faceEmbedding: face descriptor previsouly calculated on any face - * @param db: array of mapping of face descriptors to known values - * @param threshold: minimum score for matching to be considered in the result - * @returns best match - */ - // eslint-disable-next-line class-methods-use-this - match(faceEmbedding: Array, db: Array<{ name: string, source: string, embedding: number[] }>, threshold = 0): { name: string, source: string, similarity: number, embedding: number[] } { - return faceres.match(faceEmbedding, db, threshold); - } - /** Explicit backend initialization * - Normally done implicitly during initial load phase * - Call to explictly register and initialize TFJS backend without any other operations @@ -309,7 +304,7 @@ export class Human { * * @return Promise */ - async init() { + async init(): Promise { await backend.check(this, true); await this.tf.ready(); env.set(this.env); @@ -321,7 +316,7 @@ export class Human { * @param userConfig?: {@link Config} * @return Promise */ - async load(userConfig?: Partial) { + async load(userConfig?: Partial): Promise { this.state = 'load'; const timeStamp = now(); const count = Object.values(this.models).filter((model) => model).length; @@ -354,7 +349,9 @@ export class Human { // emit event /** @hidden */ - emit = (event: string) => this.events?.dispatchEvent(new Event(event)); + emit = (event: string) => { + if (this.events && this.events.dispatchEvent) this.events?.dispatchEvent(new Event(event)); + } /** Runs interpolation using last known result and returns smoothened result * Interpolation is based on time since last known result so can be called independently @@ -362,7 +359,7 @@ export class Human { * @param result?: {@link Result} optional use specific result set to run interpolation on * @returns result: {@link Result} */ - next(result: Result = this.result) { + next(result: Result = this.result): Result { return interpolate.calc(result) as Result; } diff --git a/src/image/image.ts b/src/image/image.ts index 0a104885..8a58db0a 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -9,7 +9,7 @@ import type { Config } from '../config'; import { env } from '../util/env'; import { log } from '../util/util'; -type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas; +export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas; const maxSize = 2048; // internal temp canvases diff --git a/src/models.ts b/src/models.ts index 05010bf9..41519a1f 100644 --- a/src/models.ts +++ b/src/models.ts @@ -51,13 +51,13 @@ export class Models { segmentation: null | GraphModel | Promise = null; } -export function reset(instance: Human) { +export function reset(instance: Human): void { // if (instance.config.debug) log('resetting loaded models'); for (const model of Object.keys(instance.models)) instance.models[model] = null; } /** Load method preloads all instance.configured models on-demand */ -export async function load(instance: Human) { +export async function load(instance: Human): Promise { if (env.initial) reset(instance); if (instance.config.hand.enabled) { // handpose model is a combo that must be loaded as a whole if (!instance.models.handpose && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config); @@ -87,7 +87,7 @@ export async function load(instance: Human) { } } -export async function validate(instance) { +export async function validate(instance: Human): Promise { interface Op { name: string, category: string, op: string } const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div']; for (const defined of Object.keys(instance.models)) { diff --git a/test/test-main.js b/test/test-main.js index 8f579dab..5c933c1d 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -237,25 +237,23 @@ async function test(Human, inputConfig) { const desc3 = res3 && res3.face && res3.face[0] && res3.face[0].embedding ? [...res3.face[0].embedding] : null; if (!desc1 || !desc2 || !desc3 || desc1.length !== 1024 || desc2.length !== 1024 || desc3.length !== 1024) log('error', 'failed: face descriptor', desc1?.length, desc2?.length, desc3?.length); else log('state', 'passed: face descriptor'); - res1 = Math.round(10 * human.similarity(desc1, desc2)); - res2 = Math.round(10 * human.similarity(desc1, desc3)); - res3 = Math.round(10 * human.similarity(desc2, desc3)); - if (res1 !== 5 || res2 !== 5 || res3 !== 5) log('error', 'failed: face similarity ', res1, res2, res3); - else log('state', 'passed: face similarity'); + res1 = human.similarity(desc1, desc1); + res2 = human.similarity(desc1, desc2); + res3 = human.similarity(desc1, desc3); + if (res1 < 1 || res2 < 0.9 || res3 < 0.85) log('error', 'failed: face similarity ', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); + else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] }); // test face matching log('info', 'test face matching'); - let db = []; - try { - db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString()); - } catch { /***/ } - if (db.length < 100) log('error', 'failed: face database ', db.length); + const db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString()); + const arr = db.map((rec) => rec.embedding); + if (db.length < 20) log('error', 'failed: face database ', db.length); else log('state', 'passed: face database', db.length); - res1 = human.match(desc1, db); - res2 = human.match(desc2, db); - res3 = human.match(desc3, db); - if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1, res2, res3); - else log('state', 'passed: face match', { first: { name: res1.name, similarity: res1.similarity } }, { second: { name: res2.name, similarity: res2.similarity } }, { third: { name: res3.name, similarity: res3.similarity } }); + res1 = human.match(desc1, arr); + res2 = human.match(desc2, arr); + res3 = human.match(desc3, arr); + if (res1.index !== 4 || res2.index !== 4 || res3.index !== 4) log('error', 'failed: face match ', res1, res2, res3); + else log('state', 'passed: face match', { first: { index: res1.index, similarity: res1.similarity } }, { second: { index: res2.index, similarity: res2.similarity } }, { third: { index: res3.index, similarity: res3.similarity } }); // test object detection log('info', 'test object');