ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

breaking change: new similarity and match methods

pull/280/head
Vladimir Mandic 2021-09-30 14:28:16 -04:00
parent ceaff322a8
commit b02c6fa413
9 changed files with 146 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -9,13 +9,12 @@
## Changelog
### **HEAD -> main** 2021/09/28 mandic00@live.com
### **HEAD -> main** 2021/09/29 mandic00@live.com
- release candidate
- tweaked default values
- enable handtrack as default model
- redesign face processing
### **origin/main** 2021/09/27 mandic00@live.com
- refactoring
- define app specific types
- implement box caching for movenet

21
demo/facematch/facematch.js
View File

@ -75,11 +75,12 @@ async function SelectFaceCanvas(face) {
ctx.font = 'small-caps 0.4rem "Lato"';
ctx.fillStyle = 'rgba(255, 255, 255, 1)';
}
const person = await human.match(face.embedding, db);
log('Match:', person);
const arr = db.map((rec) => rec.embedding);
const res = await human.match(face.embedding, arr);
log('Match:', db[res.index].name);
document.getElementById('desc').innerHTML = `
${face.fileName}<br>
Match: ${Math.round(1000 * person.similarity) / 10}% ${person.name}
Match: ${Math.round(1000 * res.similarity) / 10}% ${db[res.index].name}
`;
embedding = face.embedding.map((a) => parseFloat(a.toFixed(4)));
navigator.clipboard.writeText(`{"name":"unknown", "source":"${face.fileName}", "embedding":[${embedding}]},`);
@ -91,7 +92,7 @@ async function SelectFaceCanvas(face) {
for (const canvas of canvases) {
// calculate similarity from selected face to current one in the loop
const current = all[canvas.tag.sample][canvas.tag.face];
const similarity = human.similarity(face.embedding, current.embedding, 3);
const similarity = human.similarity(face.embedding, current.embedding);
// get best match
// draw the canvas
canvas.title = similarity;
@ -107,9 +108,10 @@ async function SelectFaceCanvas(face) {
// identify person
ctx.font = 'small-caps 1rem "Lato"';
const start = performance.now();
const person = await human.match(current.embedding, db);
const arr = db.map((rec) => rec.embedding);
const res = await human.match(face.embedding, arr);
time += (performance.now() - start);
if (person.similarity && person.similarity > minScore) ctx.fillText(`DB: ${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30);
if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30);
}
log('Analyzed:', 'Face:', canvases.length, 'DB:', db.length, 'Time:', time);
@ -145,9 +147,10 @@ async function AddFaceCanvas(index, res, fileName) {
ctx.font = 'small-caps 0.8rem "Lato"';
ctx.fillStyle = 'rgba(255, 255, 255, 1)';
ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
const person = await human.match(res.face[i].embedding, db);
const arr = db.map((rec) => rec.embedding);
const result = await human.match(res.face[i].embedding, arr);
ctx.font = 'small-caps 1rem "Lato"';
if (person.similarity && person.similarity > minScore) ctx.fillText(`${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30);
if (result.similarity && res.similarity > minScore) ctx.fillText(`${(100 * result.similarity).toFixed(1)}% ${db[result.index].name}`, 4, canvas.height - 30);
}
}
return ok;
@ -212,7 +215,7 @@ async function main() {
images = images.map((a) => `/human/samples/in/${a}`);
log('Adding static image list:', images);
} else {
log('Disoovered images:', images);
log('Discovered images:', images);
}
// download and analyze all images

27
src/face/faceres.ts
View File

@ -24,8 +24,6 @@ const last: Array<{
let lastCount = 0;
let skipped = Number.MAX_SAFE_INTEGER;
type DB = Array<{ name: string, source: string, embedding: number[] }>;
export async function load(config: Config): Promise<GraphModel> {
const modelUrl = join(config.modelBasePath, config.face.description?.modelPath || '');
if (env.initial) model = null;
@ -37,31 +35,6 @@ export async function load(config: Config): Promise<GraphModel> {
return model;
}
export function similarity(embedding1: Array<number>, embedding2: Array<number>, order = 2): number {
if (!embedding1 || !embedding2) return 0;
if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
if (embedding1?.length !== embedding2?.length) return 0;
// general minkowski distance, euclidean distance is limited case where order is 2
const distance = 5.0 * embedding1
.map((_val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
.reduce((sum, now) => (sum + now), 0) // sum all distances
** (1 / order); // get root of
const res = Math.max(0, 100 - distance) / 100.0;
return res;
}
export function match(embedding: Array<number>, db: DB, threshold = 0) {
let best = { similarity: 0, name: '', source: '', embedding: [] as number[] };
if (!embedding || !db || !Array.isArray(embedding) || !Array.isArray(db)) return best;
for (const f of db) {
if (f.embedding && f.name) {
const perc = similarity(embedding, f.embedding);
if (perc > threshold && perc > best.similarity) best = { ...f, similarity: perc };
}
}
return best;
}
export function enhance(input): Tensor {
const image = tf.tidy(() => {
// input received from detector is already normalized to 0..1

60
src/face/match.ts Normal file
View File

@ -0,0 +1,60 @@
/** Defines Descriptor type */
export type Descriptor = Array<number>
/** Calculates distance between two descriptors
* - Minkowski distance algorithm of nth order if `order` is different than 2
* - Euclidean distance if `order` is 2 (default)
*
* Options:
* - `order`
*
* Note: No checks are performed for performance reasons so make sure to pass valid number arrays of equal length
*/
export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) {
// general minkowski distance, euclidean distance is limited case where order is 2
let sum = 0;
for (let i = 0; i < descriptor1.length; i++) {
const diff = (options.order === 2) ? (descriptor1[i] - descriptor2[i]) : (Math.abs(descriptor1[i] - descriptor2[i]));
sum += (options.order === 2) ? (diff * diff) : (diff ** options.order);
}
return sum;
}
/** Calculates normalized similarity between two descriptors based on their `distance`
*/
export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2 }) {
const dist = distance(descriptor1, descriptor2, options);
const invert = (options.order === 2) ? Math.sqrt(dist) : dist ** (1 / options.order);
return Math.max(0, 100 - invert) / 100.0;
}
/** Matches given descriptor to a closest entry in array of descriptors
* @param descriptor face descriptor
* @param descriptors array of face descriptors to commpare given descriptor to
*
* Options:
* - `order` see {@link distance} method
* - `threshold` match will return result first result for which {@link distance} is below `threshold` even if there may be better results
*
* @returns object with index, distance and similarity
* - `index` index array index where best match was found or -1 if no matches
* - {@link distance} calculated `distance` of given descriptor to the best match
* - {@link similarity} calculated normalized `similarity` of given descriptor to the best match
*/
export function match(descriptor: Descriptor, descriptors: Array<Descriptor>, options = { order: 2, threshold: 0 }) {
if (!Array.isArray(descriptor) || !Array.isArray(descriptors) || descriptor.length < 64 || descriptors.length === 0 || descriptor.length !== descriptors[0].length) { // validate input
return { index: -1, distance: Number.POSITIVE_INFINITY, similarity: 0 };
}
let best = Number.MAX_SAFE_INTEGER;
let index = -1;
for (let i = 0; i < descriptors.length; i++) {
const res = distance(descriptor, descriptors[i], { order: options.order });
if (res < best) {
best = res;
index = i;
}
if (best < options.threshold) break;
}
best = (options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order);
return { index, distance: best, similarity: Math.max(0, 100 - best) / 100.0 };
}

109
src/human.ts
View File

@ -2,49 +2,64 @@
* Human main module
*/
// module imports
import { log, now, mergeDeep, validate } from './util/util';
import { Config, defaults } from './config';
import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
import { defaults } from './config';
import * as tf from '../dist/tfjs.esm.js';
import * as models from './models';
import * as app from '../package.json';
import * as backend from './tfjs/backend';
// import * as blazepose from './body/blazepose-v1';
import * as blazepose from './body/blazepose';
import * as centernet from './object/centernet';
import * as draw from './util/draw';
import * as efficientpose from './body/efficientpose';
import * as env from './util/env';
import * as face from './face/face';
import * as facemesh from './face/facemesh';
import * as faceres from './face/faceres';
import * as posenet from './body/posenet';
import * as handtrack from './hand/handtrack';
import * as gesture from './gesture/gesture';
import * as handpose from './handpose/handpose';
// import * as blazepose from './body/blazepose-v1';
import * as blazepose from './body/blazepose';
import * as efficientpose from './body/efficientpose';
import * as handtrack from './hand/handtrack';
import * as humangl from './tfjs/humangl';
import * as image from './image/image';
import * as interpolate from './util/interpolate';
import * as match from './face/match';
import * as models from './models';
import * as movenet from './body/movenet';
import * as nanodet from './object/nanodet';
import * as centernet from './object/centernet';
import * as segmentation from './segmentation/segmentation';
import * as gesture from './gesture/gesture';
import * as image from './image/image';
import * as draw from './util/draw';
import * as persons from './util/persons';
import * as interpolate from './util/interpolate';
import * as env from './util/env';
import * as backend from './tfjs/backend';
import * as humangl from './tfjs/humangl';
import * as app from '../package.json';
import * as posenet from './body/posenet';
import * as segmentation from './segmentation/segmentation';
import * as warmups from './warmup';
// type definitions
import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result';
import type { Tensor } from './tfjs/types';
import type { DrawOptions } from './util/draw';
import type { Input } from './image/image';
import type { Config } from './config';
// export types
/** Defines configuration options used by all **Human** methods */
export * from './config';
/** Defines result types returned by all **Human** methods */
export * from './result';
/** Defines DrawOptions used by `human.draw.*` methods */
export type { DrawOptions } from './util/draw';
export { env, Env } from './util/env';
/** Face descriptor type as number array */
export type { Descriptor } from './face/match';
/** Box and Point primitives */
export { Box, Point } from './result';
/** Defines all possible models used by **Human** library */
export { Models } from './models';
/** Defines all possible input types for **Human** detection
* @typedef Input Type
*/
export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;
/** Defines all possible input types for **Human** detection */
export { Input } from './image/image';
/** Events dispatched by `human.events`
*
@ -139,7 +154,7 @@ export class Human {
* - `warmup`: triggered when warmup is complete
* - `error`: triggered on some errors
*/
events: EventTarget;
events: EventTarget | undefined;
/** Reference face triangualtion array of 468 points, used for triangle references between points */
faceTriangulation: typeof facemesh.triangulation;
/** Refernce UV map of 468 values, used for 3D mapping of the face mesh */
@ -176,7 +191,7 @@ export class Human {
this.#analyzeMemoryLeaks = false;
this.#checkSanity = false;
this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 };
this.events = new EventTarget();
this.events = (typeof EventTarget !== 'undefined') ? new EventTarget() : undefined;
// object that contains all initialized models
this.models = new models.Models();
// reexport draw methods
@ -230,17 +245,22 @@ export class Human {
}
/** Reset configuration to default values */
reset() {
reset(): void {
const currentBackend = this.config.backend; // save backend;
this.config = JSON.parse(JSON.stringify(defaults));
this.config.backend = currentBackend;
}
/** Validate current configuration schema */
validate(userConfig?: Partial<Config>) {
public validate(userConfig?: Partial<Config>) {
return validate(defaults, userConfig || this.config);
}
/** Exports face matching methods */
public similarity = match.similarity;
public distance = match.distance;
public match = match.match;
/** Process input as return canvas and tensor
*
* @param input: {@link Input}
@ -250,19 +270,6 @@ export class Human {
return image.process(input, this.config);
}
/** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings)
* - Calculation is based on normalized Minkowski distance between two descriptors
* - Default is Euclidean distance which is Minkowski distance of 2nd order
*
* @param embedding1: face descriptor as array of numbers
* @param embedding2: face descriptor as array of numbers
* @returns similarity: number
*/
// eslint-disable-next-line class-methods-use-this
similarity(embedding1: Array<number>, embedding2: Array<number>): number {
return faceres.similarity(embedding1, embedding2);
}
/** Segmentation method takes any input and returns processed canvas with body segmentation
* - Optional parameter background is used to fill the background with specific input
* - Segmentation is not triggered as part of detect process
@ -290,18 +297,6 @@ export class Human {
return faceres.enhance(input);
}
/** Math method find best match between provided face descriptor and predefined database of known descriptors
*
* @param faceEmbedding: face descriptor previsouly calculated on any face
* @param db: array of mapping of face descriptors to known values
* @param threshold: minimum score for matching to be considered in the result
* @returns best match
*/
// eslint-disable-next-line class-methods-use-this
match(faceEmbedding: Array<number>, db: Array<{ name: string, source: string, embedding: number[] }>, threshold = 0): { name: string, source: string, similarity: number, embedding: number[] } {
return faceres.match(faceEmbedding, db, threshold);
}
/** Explicit backend initialization
* - Normally done implicitly during initial load phase
* - Call to explictly register and initialize TFJS backend without any other operations
@ -309,7 +304,7 @@ export class Human {
*
* @return Promise<void>
*/
async init() {
async init(): Promise<void> {
await backend.check(this, true);
await this.tf.ready();
env.set(this.env);
@ -321,7 +316,7 @@ export class Human {
* @param userConfig?: {@link Config}
* @return Promise<void>
*/
async load(userConfig?: Partial<Config>) {
async load(userConfig?: Partial<Config>): Promise<void> {
this.state = 'load';
const timeStamp = now();
const count = Object.values(this.models).filter((model) => model).length;
@ -354,7 +349,9 @@ export class Human {
// emit event
/** @hidden */
emit = (event: string) => this.events?.dispatchEvent(new Event(event));
emit = (event: string) => {
if (this.events && this.events.dispatchEvent) this.events?.dispatchEvent(new Event(event));
}
/** Runs interpolation using last known result and returns smoothened result
* Interpolation is based on time since last known result so can be called independently
@ -362,7 +359,7 @@ export class Human {
* @param result?: {@link Result} optional use specific result set to run interpolation on
* @returns result: {@link Result}
*/
next(result: Result = this.result) {
next(result: Result = this.result): Result {
return interpolate.calc(result) as Result;
}

2
src/image/image.ts
View File

@ -9,7 +9,7 @@ import type { Config } from '../config';
import { env } from '../util/env';
import { log } from '../util/util';
type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas;
export type Input = Tensor | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas | typeof Image | typeof env.Canvas;
const maxSize = 2048;
// internal temp canvases

6
src/models.ts
View File

@ -51,13 +51,13 @@ export class Models {
segmentation: null | GraphModel | Promise<GraphModel> = null;
}
export function reset(instance: Human) {
export function reset(instance: Human): void {
// if (instance.config.debug) log('resetting loaded models');
for (const model of Object.keys(instance.models)) instance.models[model] = null;
}
/** Load method preloads all instance.configured models on-demand */
export async function load(instance: Human) {
export async function load(instance: Human): Promise<void> {
if (env.initial) reset(instance);
if (instance.config.hand.enabled) { // handpose model is a combo that must be loaded as a whole
if (!instance.models.handpose && instance.config.hand.detector?.modelPath?.includes('handdetect')) [instance.models.handpose, instance.models.handskeleton] = await handpose.load(instance.config);
@ -87,7 +87,7 @@ export async function load(instance: Human) {
}
}
export async function validate(instance) {
export async function validate(instance: Human): Promise<void> {
interface Op { name: string, category: string, op: string }
const simpleOps = ['const', 'placeholder', 'noop', 'pad', 'squeeze', 'add', 'sub', 'mul', 'div'];
for (const defined of Object.keys(instance.models)) {

3
src/segmentation/segmentation.ts
View File

@ -12,8 +12,7 @@ import * as image from '../image/image';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;
import type { Input } from '../image/image';
let model: GraphModel;
let busy = false;

28
test/test-main.js
View File

@ -237,25 +237,23 @@ async function test(Human, inputConfig) {
const desc3 = res3 && res3.face && res3.face[0] && res3.face[0].embedding ? [...res3.face[0].embedding] : null;
if (!desc1 || !desc2 || !desc3 || desc1.length !== 1024 || desc2.length !== 1024 || desc3.length !== 1024) log('error', 'failed: face descriptor', desc1?.length, desc2?.length, desc3?.length);
else log('state', 'passed: face descriptor');
res1 = Math.round(10 * human.similarity(desc1, desc2));
res2 = Math.round(10 * human.similarity(desc1, desc3));
res3 = Math.round(10 * human.similarity(desc2, desc3));
if (res1 !== 5 || res2 !== 5 || res3 !== 5) log('error', 'failed: face similarity ', res1, res2, res3);
else log('state', 'passed: face similarity');
res1 = human.similarity(desc1, desc1);
res2 = human.similarity(desc1, desc2);
res3 = human.similarity(desc1, desc3);
if (res1 < 1 || res2 < 0.9 || res3 < 0.85) log('error', 'failed: face similarity ', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
// test face matching
log('info', 'test face matching');
let db = [];
try {
db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString());
} catch { /***/ }
if (db.length < 100) log('error', 'failed: face database ', db.length);
const db = JSON.parse(fs.readFileSync('demo/facematch/faces.json').toString());
const arr = db.map((rec) => rec.embedding);
if (db.length < 20) log('error', 'failed: face database ', db.length);
else log('state', 'passed: face database', db.length);
res1 = human.match(desc1, db);
res2 = human.match(desc2, db);
res3 = human.match(desc3, db);
if (!res1 || !res1['name'] || !res2 || !res2['name'] || !res3 || !res3['name']) log('error', 'failed: face match ', res1, res2, res3);
else log('state', 'passed: face match', { first: { name: res1.name, similarity: res1.similarity } }, { second: { name: res2.name, similarity: res2.similarity } }, { third: { name: res3.name, similarity: res3.similarity } });
res1 = human.match(desc1, arr);
res2 = human.match(desc2, arr);
res3 = human.match(desc3, arr);
if (res1.index !== 4 || res2.index !== 4 || res3.index !== 4) log('error', 'failed: face match ', res1, res2, res3);
else log('state', 'passed: face match', { first: { index: res1.index, similarity: res1.similarity } }, { second: { index: res2.index, similarity: res2.similarity } }, { third: { index: res3.index, similarity: res3.similarity } });
// test object detection
log('info', 'test object');