distance based on minkowski space and limited euclidean space

CHANGELOG.md

@@ -9,6 +9,9 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
+### **HEAD -> main** 2021/03/12 mandic00@live.com
+
+
 ### **1.1.1** 2021/03/12 mandic00@live.com
 
 - switched face embedding to mobileface

TODO.md

@@ -1,7 +1,25 @@
 # To-Do list for Human library
 
-- Strong typing
-- Automated testing
-- Explore EfficientPose  
+## Big Ticket Items
+
+- Strong(er) typing
+- Automated testing framework
+
+## Explore Models
+
+- EfficientPose  
   <https://github.com/daniegr/EfficientPose>  
   <https://github.com/PINTO0309/PINTO_model_zoo/tree/main/084_EfficientPose>
+- ArcFace
+- RetinaFace
+- CenterFace
+
+## WiP Items
+
+- Embedding:
+  - Try average of flipped image
+  - Try with variable aspect ratio
+
+## Issues
+
+*N/A*

demo/embedding.js

@@ -24,7 +24,7 @@ const userConfig = {
 
 const human = new Human(userConfig); // new instance of human
 
-const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'];
+// const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'];
 // const samples = ['../assets/sample-me.jpg', '../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg',
 //   '../private/me (1).jpg', '../private/me (2).jpg', '../private/me (3).jpg', '../private/me (4).jpg', '../private/me (5).jpg', '../private/me (6).jpg', '../private/me (7).jpg', '../private/me (8).jpg',
 //   '../private/me (9).jpg', '../private/me (10).jpg', '../private/me (11).jpg', '../private/me (12).jpg', '../private/me (13).jpg'];
@@ -57,7 +57,7 @@ async function analyze(face) {
   const canvases = document.getElementsByClassName('face');
   for (const canvas of canvases) {
     // calculate simmilarity from selected face to current one in the loop
-    const res = human.simmilarity(face.embedding, all[canvas.tag.sample][canvas.tag.face].embedding);
+    const res = human.simmilarity(face.embedding, all[canvas.tag.sample][canvas.tag.face].embedding, 3);
     // draw the canvas and simmilarity score
     canvas.title = res;
     await human.tf.browser.toPixels(all[canvas.tag.sample][canvas.tag.face].tensor, canvas);
@@ -98,8 +98,8 @@ async function faces(index, res) {
   }
 }
 
-async function add(index) {
-  log('Add image:', samples[index]);
+async function add(index, image) {
+  log('Add image:', index + 1, image);
   return new Promise((resolve) => {
     const img = new Image(100, 100);
     img.onload = () => { // must wait until image is loaded
@@ -107,14 +107,27 @@ async function add(index) {
       document.getElementById('images').appendChild(img); // and finally we can add it
       resolve(true);
     };
-    img.title = samples[index];
-    img.src = samples[index];
+    img.title = image;
+    img.src = encodeURI(image);
   });
 }
 
 async function main() {
   await human.load();
-  for (const i in samples) await add(i); // download and analyze all images
+  // enumerate all sample images in /assets
+  let res = await fetch('/assets');
+  let dir = (res && res.ok) ? await res.json() : [];
+  let images = dir.filter((img) => (img.endsWith('.jpg') && img.includes('sample')));
+
+  // enumerate additional private test images in /private, not includded in git repository
+  res = await fetch('/private');
+  dir = (res && res.ok) ? await res.json() : [];
+  images = images.concat(dir.filter((img) => (img.endsWith('.jpg'))));
+
+  // download and analyze all images
+  log('Enumerated:', images.length, 'images');
+  for (const i in images) await add(i, images[i]);
+
   log('Ready');
 }
 

server/serve.js

@@ -85,18 +85,22 @@ async function watch() {
 
 // get file content for a valid url request
 function handle(url) {
-  return new Promise((resolve) => {
-    let obj = { ok: false };
-    obj.file = decodeURI(url);
-    if (!fs.existsSync(obj.file)) resolve(null);
-    obj.stat = fs.statSync(obj.file);
-    if (obj.stat.isFile()) obj.ok = true;
-    if (!obj.ok && obj.stat.isDirectory()) {
-      obj.file = path.join(obj.file, options.default);
-      // @ts-ignore
-      obj = handle(obj.file);
+  return new Promise(async (resolve) => {
+    let obj = { ok: false, file: decodeURI(url) };
+    if (!fs.existsSync(obj.file)) {
+      resolve(obj);
+    } else {
+      obj.stat = fs.statSync(obj.file);
+      if (obj.stat.isFile()) obj.ok = true;
+      if (!obj.ok && obj.stat.isDirectory()) {
+        if (fs.existsSync(path.join(obj.file, options.default))) {
+          obj = await handle(path.join(obj.file, options.default));
+        } else {
+          obj.ok = obj.stat.isDirectory();
+        }
+      }
+      resolve(obj);
     }
-    resolve(obj);
   });
 }
 
@@ -106,33 +110,42 @@ async function httpRequest(req, res) {
     // get original ip of requestor, regardless if it's behind proxy or not
     const forwarded = (req.headers['forwarded'] || '').match(/for="\[(.*)\]:/);
     const ip = (Array.isArray(forwarded) ? forwarded[1] : null) || req.headers['x-forwarded-for'] || req.ip || req.socket.remoteAddress;
-    if (!result || !result.ok) {
-      res.writeHead(404, { 'Content-Type': 'text/html' });
+    if (!result || !result.ok || !result.stat) {
+      res.writeHead(404, { 'Content-Type': 'text/html; charset=utf-8' });
       res.end('Error 404: Not Found\n', 'utf-8');
       log.warn(`${req.method}/${req.httpVersion}`, res.statusCode, req.url, ip);
     } else {
-      const ext = String(path.extname(result.file)).toLowerCase();
-      const contentType = mime[ext] || 'application/octet-stream';
-      const accept = req.headers['accept-encoding'] ? req.headers['accept-encoding'].includes('br') : false; // does target accept brotli compressed data
-      res.writeHead(200, {
-        // 'Content-Length': result.stat.size, // not using as it's misleading for compressed streams
-        'Content-Language': 'en', 'Content-Type': contentType, 'Content-Encoding': accept ? 'br' : '', 'Last-Modified': result.stat.mtime, 'Cache-Control': 'no-cache', 'X-Content-Type-Options': 'nosniff',
-      });
-      const compress = zlib.createBrotliCompress({ params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 5 } }); // instance of brotli compression with level 5
-      const stream = fs.createReadStream(result.file);
-      if (!accept) stream.pipe(res); // don't compress data
-      else stream.pipe(compress).pipe(res); // compress data
+      if (result?.stat?.isFile()) {
+        const ext = String(path.extname(result.file)).toLowerCase();
+        const contentType = mime[ext] || 'application/octet-stream';
+        const accept = req.headers['accept-encoding'] ? req.headers['accept-encoding'].includes('br') : false; // does target accept brotli compressed data
+        res.writeHead(200, {
+          // 'Content-Length': result.stat.size, // not using as it's misleading for compressed streams
+          'Content-Language': 'en', 'Content-Type': contentType, 'Content-Encoding': accept ? 'br' : '', 'Last-Modified': result.stat.mtime, 'Cache-Control': 'no-cache', 'X-Content-Type-Options': 'nosniff',
+        });
+        const compress = zlib.createBrotliCompress({ params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 5 } }); // instance of brotli compression with level 5
+        const stream = fs.createReadStream(result.file);
+        if (!accept) stream.pipe(res); // don't compress data
+        else stream.pipe(compress).pipe(res); // compress data
 
-      // alternative methods of sending data
-      /// 2. read stream and send by chunk
-      // const stream = fs.createReadStream(result.file);
-      // stream.on('data', (chunk) => res.write(chunk));
-      // stream.on('end', () => res.end());
+        // alternative methods of sending data
+        /// 2. read stream and send by chunk
+        // const stream = fs.createReadStream(result.file);
+        // stream.on('data', (chunk) => res.write(chunk));
+        // stream.on('end', () => res.end());
 
-      // 3. read entire file and send it as blob
-      // const data = fs.readFileSync(result.file);
-      // res.write(data);
-      log.data(`${req.method}/${req.httpVersion}`, res.statusCode, contentType, result.stat.size, req.url, ip);
+        // 3. read entire file and send it as blob
+        // const data = fs.readFileSync(result.file);
+        // res.write(data);
+        log.data(`${req.method}/${req.httpVersion}`, res.statusCode, contentType, result.stat.size, req.url, ip);
+      }
+      if (result?.stat?.isDirectory()) {
+        res.writeHead(200, { 'Content-Language': 'en', 'Content-Type': 'application/json; charset=utf-8', 'Last-Modified': result.stat.mtime, 'Cache-Control': 'no-cache', 'X-Content-Type-Options': 'nosniff' });
+        let dir = fs.readdirSync(result.file);
+        dir = dir.map((f) => '/' + path.join(path.basename(result.file), f));
+        res.end(JSON.stringify(dir), 'utf-8');
+        log.data(`${req.method}/${req.httpVersion}`, res.statusCode, 'directory/json', result.stat.size, req.url, ip);
+      }
     }
   });
 }

src/blazeface/facemesh.ts

@@ -32,26 +32,15 @@ export class MediaPipeFaceMesh {
       const box = prediction.box ? [
         Math.max(0, prediction.box.startPoint[0]),
         Math.max(0, prediction.box.startPoint[1]),
-        Math.min(input.shape[2], prediction.box.endPoint[0]) - prediction.box.startPoint[0],
-        Math.min(input.shape[1], prediction.box.endPoint[1]) - prediction.box.startPoint[1],
+        Math.min(input.shape[1], prediction.box.endPoint[0]) - prediction.box.startPoint[0],
+        Math.min(input.shape[2], prediction.box.endPoint[1]) - prediction.box.startPoint[1],
       ] : 0;
       const boxRaw = prediction.box ? [
         Math.max(0, prediction.box.startPoint[0] / input.shape[2]),
         Math.max(0, prediction.box.startPoint[1] / input.shape[1]),
-        Math.min(input.shape[2], (prediction.box.endPoint[0]) - prediction.box.startPoint[0]) / input.shape[2],
-        Math.min(input.shape[1], (prediction.box.endPoint[1]) - prediction.box.startPoint[1]) / input.shape[1],
+        Math.min(input.shape[1], (prediction.box.endPoint[0]) - prediction.box.startPoint[0]) / input.shape[2],
+        Math.min(input.shape[2], (prediction.box.endPoint[1]) - prediction.box.startPoint[1]) / input.shape[1],
       ] : [];
-      let offsetRaw = <any>[];
-      if (meshRaw.length > 0 && boxRaw.length > 0) {
-        const dimX = meshRaw.map((pt) => pt[0]);
-        const dimY = meshRaw.map((pt) => pt[1]);
-        offsetRaw = [
-          Math.max(0, 0 + Math.min(...dimY) - boxRaw[0]), // distance of detected face border to box top edge
-          Math.max(0, 0 + Math.min(...dimX) - boxRaw[1]), // distance of detected face border to box left edge
-          Math.min(1, 1 - Math.max(...dimY) + boxRaw[2]), // distance of detected face border to box bottom edge
-          Math.min(1, 1 - Math.max(...dimX) + boxRaw[3]), // distance of detected face border to box right edge
-        ];
-      }
       results.push({
         confidence: prediction.faceConfidence || prediction.boxConfidence || 0,
         boxConfidence: prediction.boxConfidence,
@@ -60,7 +49,6 @@ export class MediaPipeFaceMesh {
         mesh,
         boxRaw,
         meshRaw,
-        offsetRaw,
         annotations,
         image: prediction.image ? tf.clone(prediction.image) : null,
       });

src/embedding/embedding.ts

@@ -16,13 +16,12 @@ export function simmilarity(embedding1, embedding2, order = 2) {
   if (!embedding1 || !embedding2) return 0;
   if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
   if (embedding1?.length !== embedding2?.length) return 0;
-  // general minkowski distance
-  // euclidean distance is limited case where order is 2
+  // general minkowski distance, euclidean distance is limited case where order is 2
   const distance = embedding1
     .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
     .reduce((sum, now) => (sum + now), 0) // sum all distances
     ** (1 / order); // get root of
-  const res = Math.max(Math.trunc(1000 * (1 - (1 * distance))) / 1000, 0);
+  const res = Math.max(Math.trunc(1000 * (1 - distance)) / 1000, 0);
   return res;
 }
 
@@ -33,14 +32,10 @@ export function enhance(input) {
     // const data = tf.image.resizeBilinear(input, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); // just resize to fit the embedding model
 
     // do a tight crop of image and resize it to fit the model
-    // maybe offsets are already prepared by face model, if not use empirical values
-    const box = input.offsetRaw
-      ? [input.offsetRaw] // crop based on face mesh borders
-      : [[0.05, 0.15, 0.85, 0.85]]; // fixed crop for top, left, bottom, right
-    console.log('BOX', box[0]);
+    const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
     const tensor = input.image || input.tensor;
-    const crop = tensor.shape.length === 3
-      ? tf.image.cropAndResize(tensor.expandDims(0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch if missing
+    const crop = (tensor.shape.length === 3)
+      ? tf.image.cropAndResize(tensor.expandDims(0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
       : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
 
     // convert to black&white to avoid colorization impact
@@ -77,9 +72,9 @@ export async function predict(input, config) {
           const scale = res.div(l2);
           return scale;
         });
+        tf.dispose(scaled);
         */
         data = res.dataSync();
-        // tf.dispose(scaled);
         tf.dispose(res);
       } else {
         const profileData = await tf.profile(() => model.predict({ img_inputs: image }));

src/human.ts

@@ -394,17 +394,6 @@ class Human {
       // combine results
       faceRes.push({
         ...face,
-        /*
-        confidence: face.confidence,
-        faceConfidence: face.faceConfidence,
-        boxConfidence: face.boxConfidence,
-        box: face.box,
-        mesh: face.mesh,
-        boxRaw: face.boxRaw,
-        meshRaw: face.meshRaw,
-        offsetRaw: face.offsetRaw,
-        annotations: face.annotations,
-        */
         age: ageRes.age,
         gender: genderRes.gender,
         genderConfidence: genderRes.confidence,

wiki

@@ -1 +1 @@
-Subproject commit cef8c9cc9f09f8709cd4bd1daff817a59df4b4d1
+Subproject commit ac5d01255f2f02de6b308b82976c5866c458f149