From 3cc1daff22fc725f9fc8dd83a17a0f7152d1b714 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 30 Sep 2021 14:28:26 -0400 Subject: [PATCH] document new similarity and match methods --- Embedding.md | 62 +++++++++++++++++++++++----------------------------- Usage.md | 8 +++---- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/Embedding.md b/Embedding.md index ede2c84..c6bae8f 100644 --- a/Embedding.md +++ b/Embedding.md @@ -15,13 +15,13 @@ It highlights functionality such as: ## Usage -To use face simmilaity compare feature, you must first enable `face.description` module -and calculate embedding vectors for both first and second image you want to compare +To use face similarity compare feature, you must first enable `face.description` module +to calculate embedding vectors *(a.k.a. face descriptors)* for both all images you want to compare -To achieve quality results, it is also highly recommended to have `face.mesh` and `face.detection.rotation` -enabled as calculating feature vectors on non-quality inputs can lead to false results +To achieve accurate results, it is also highly recommended to have `face.mesh` and `face.detection.rotation` +enabled as calculating feature vectors on low-quality inputs can lead to false results -Similarity match above 65% is considered good match while similarity match above 55% is considered best-guess +Similarity match above **65%** is considered good match while similarity match above **55%** is considered best-guess For example, @@ -36,21 +36,18 @@ const myConfig = { }; const human = new Human(myConfig); - const firstResult = await human.detect(firstImage); const secondResult = await human.detect(secondImage); - const similarity = human.similarity(firstResult.face[0].embedding, secondResult.face[0].embedding); - console.log(`faces are ${100 * similarity}% simmilar`); ``` If the image or video frame have multiple faces and you want to match all of them, simply loop through all `results.face` ```js -for (let i = 0; i < secondResult.face.length; i++) { - const secondEmbedding = secondResult.face[i].embedding; - const similarity = human.similarity(firstEmbedding, secondEmbedding); +for (let i = 0; i < currentResult.face.length; i++) { + const currentEmbedding = currentResult.face[i].embedding; + const similarity = human.similarity(referenceEmbedding, currentEmbedding); console.log(`face ${i} is ${100 * similarity}% simmilar`); } ``` @@ -77,7 +74,7 @@ They can be stored as normal arrays and reused as needed ## Face Similarity Similarity function is based on general *Minkowski distance* between all points in vector -*[Minkowski distance](https://en.wikipedia.org/wiki/Minkowski_distance) is a nth root of sum of nth powers of distances between each point (each value in 192-member array)* +*[Minkowski distance](https://en.wikipedia.org/wiki/Minkowski_distance) is a nth root of sum of nth powers of distances between each point (each descriptor is 1024-member array)* *Default is Eucliean distance which is a limited case of Minkowski distance with order of 2* @@ -85,18 +82,10 @@ Changing `order` can make similarity matching more or less sensitive (default or For example, those will produce slighly different results: ```js - const similarity2ndOrder = human.similarity(firstEmbedding, secondEmbedding, 2); - const similarity3rdOrder = human.similarity(firstEmbedding, secondEmbedding, 3); + const similarity2ndOrder = human.similarity(firstEmbedding, secondEmbedding, { order = 2 }); + const similarity3rdOrder = human.similarity(firstEmbedding, secondEmbedding, { order = 3 }); ``` -How similarity is calculated: - -```js - const distance = ((firstEmbedding.map((val, i) => (val - secondEmbedding[i])).reduce((dist, diff) => dist + (diff ** order), 0) ** (1 / order))); -``` - -*Once embedding values are calculated and stored, if you want to use stored embedding values without requiring `Human` library you can use above formula to calculate similarity on the fly* -
## Face Recognition @@ -104,37 +93,40 @@ How similarity is calculated: Once you run face embedding analysis, you can store results in an annotated form to be used at the later time to find the best match for any given face -Format of annotated database is: +For example: ```js - const db = [ - { name: 'person a', source: 'optional-tag', embedding: [...]} - { name: 'person b', source: 'optional-tag', embedding: [...]} - ... - ] + const db = []; + const res = await human.detect(input); + db.push({ label: 'this-is-me', embedding: res.face[0].embedding }); ``` -where embedding is a result received in `face.embedding` after running detection - Note that you can have multiple entries for the same person and best match will be used -To find the best match, simply use function while providing embedding descriptor to compare and pre-prepared database -Last parameter is optional and notes a minimal threshold for a match +To find the best match, simply use `match` method while providing embedding descriptor to compare and pre-prepared list of descriptors ```js - const best = human.match(current.embedding, db, 0) - // return is object: { name: 'person a', similarity '0.99', source 'some-image-file' } + const embeddingArray = db.map((record) => record.embedding); // build array with just embeddings + const best = human.match(embedding, embeddingArray); // return is object: { index: number, similarity: number, distance: number } + const label = embeddingArray[best.index].label; + console.log({ name, similarity: best.similarity }); ``` Database can be further stored in a JS or JSON file and retrieved when needed to have a permanent database of faces that can be expanded over time to cover any number of known faces For example, see `/demo/facematch/facematch.js` and example database `/demo/facematch/faces.json`: +> download db with known faces using http/https ```js - // download db with known faces let res = await fetch('/demo/facematch/faces.json'); db = (res && res.ok) ? await res.json() : []; ``` +> download db with known faces from a local file +```js + const fs = require('fs'); + const buffer = fs.readFileSync('/demo/facematch/faces.json'); + db = JSON.parse(buffer); +```
diff --git a/Usage.md b/Usage.md index 97b1c28..103ebd7 100644 --- a/Usage.md +++ b/Usage.md @@ -74,10 +74,10 @@ Additional functions used for face recognition: For details, see [embedding documentation](https://github.com/vladmandic/human/wiki/Embedding) ```js - human.similarity(embedding1, embedding2) // runs similarity calculation between two provided embedding vectors - // vectors for source and target must be previously detected using - // face.description module - human.match(embedding, db, threshold) // finds best match for current face in a provided list of faces + human.similarity(descriptor1, descriptor2) // runs similarity calculation between two provided embedding vectors + // vectors for source and target must be previously detected using + // face.description module + human.match(descriptor, descriptors) // finds best match for current face in a provided list of faces human.enhance(face) // returns enhanced tensor of a previously detected face that can be used for visualizations ```