From 54b492b987f69fbfc23c0c999d97bf47ed5d6f3d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 16 Nov 2021 13:07:44 -0500 Subject: [PATCH] add extra face rotation prior to mesh --- CHANGELOG.md | 3 ++- TODO.md | 9 +++++---- demo/faceid/README.md | 7 +++++++ demo/facematch/facematch.js | 15 +++++++-------- demo/typescript/index.js | 2 +- demo/typescript/index.ts | 4 ++-- src/face/blazeface.ts | 13 ++++--------- src/face/facemesh.ts | 7 +++++-- src/face/facemeshutil.ts | 4 ++-- src/image/image.ts | 2 +- 10 files changed, 36 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61f1993b..6ee3d0d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ ## Changelog -### **HEAD -> main** 2021/11/14 mandic00@live.com +### **release 2.5.2** 2021/11/15 mandic00@live.com +- improve error handling ### **2.5.2** 2021/11/14 mandic00@live.com diff --git a/TODO.md b/TODO.md index 0d55d65e..52ba6c63 100644 --- a/TODO.md +++ b/TODO.md @@ -6,9 +6,9 @@ ### Exploring -- Optical Flow: -- Histogram Equalization: Regular, Adaptive, Contrast Limited, CLAHE -- TFLite Models: +- Optical flow: +- Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE +- TFLite models: - Body segmentation: `robust-video-matting` - TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch @@ -19,6 +19,7 @@ #### WebGPU Experimental support only until support is officially added in Chromium + - Performance issues: @@ -39,7 +40,7 @@ MoveNet MultiPose model does not work with WASM backend due to missing F32 broad

-## Pending release notes: +## Pending Release Notes New: - new demo `demos/faceid` that utilizes multiple algorithm to validate input before triggering face recognition diff --git a/demo/faceid/README.md b/demo/faceid/README.md index bbaee95a..b3c0a071 100644 --- a/demo/faceid/README.md +++ b/demo/faceid/README.md @@ -32,3 +32,10 @@ designed to serve as a quick check when used together with other indicators: - Checks if input has obvious artifacts due to recording (e.g. playing back phone recording of a face) - Configuration: `human.config.face.liveness`.enabled - Result: `human.result.face[0].live` as score + +### Models + +**FaceID** is compatible with +- `faceres.json` (default) perfoms combined age/gender/descriptor analysis +- `faceres-deep.json` higher resolution variation of `faceres` +- `mobilefacenet` alternative model for face descriptor analysis diff --git a/demo/facematch/facematch.js b/demo/facematch/facematch.js index b7945963..d7d3ca5e 100644 --- a/demo/facematch/facematch.js +++ b/demo/facematch/facematch.js @@ -19,10 +19,12 @@ const userConfig = { filter: { enabled: true, equalization: true, + width: 0, }, face: { enabled: true, - detector: { rotation: true, return: true, maxDetected: 50 }, + // detector: { rotation: false, return: true, maxDetected: 50, iouThreshold: 0.206, minConfidence: 0.122 }, + detector: { return: true, rotation: true, maxDetected: 50, iouThreshold: 0.01, minConfidence: 0.2 }, mesh: { enabled: true }, iris: { enabled: false }, emotion: { enabled: true }, @@ -138,7 +140,8 @@ async function SelectFaceCanvas(face) { async function AddFaceCanvas(index, res, fileName) { all[index] = res.face; for (const i in res.face) { - if (res.face[i].mesh.length === 0 || !res.face[i].tensor) continue; // did not get valid results + if (!res.face[i].tensor) continue; // did not get valid results + if ((res.face[i].faceScore || 0) < human.config.face.detector.minConfidence) continue; // face analysis score too low all[index][i].fileName = fileName; const canvas = document.createElement('canvas'); canvas.tag = { sample: index, face: i, source: fileName }; @@ -177,9 +180,9 @@ async function AddImageElement(index, image, length) { return new Promise((resolve) => { const img = new Image(128, 128); img.onload = () => { // must wait until image is loaded + document.getElementById('images').appendChild(img); // and finally we can add it human.detect(img, userConfig).then((res) => { AddFaceCanvas(index, res, image); // then wait until image is analyzed - document.getElementById('images').appendChild(img); // and finally we can add it resolve(true); }); }; @@ -236,12 +239,8 @@ async function main() { log('Discovered images:', images); } - // images = ['/samples/in/solvay1927.jpg']; + // images = ['/samples/in/person-lexi.jpg', '/samples/in/person-carolina.jpg', '/samples/in/solvay1927.jpg']; - // download and analyze all images - // const promises = []; - // for (let i = 0; i < images.length; i++) promises.push(AddImageElement(i, images[i], images.length)); - // await Promise.all(promises); const t0 = human.now(); for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length); const t1 = human.now(); diff --git a/demo/typescript/index.js b/demo/typescript/index.js index 84afb663..7185752d 100644 --- a/demo/typescript/index.js +++ b/demo/typescript/index.js @@ -8,7 +8,7 @@ import { Human } from "../../dist/human.esm.js"; var humanConfig = { modelBasePath: "../../models", - filter: { equalization: false } + filter: { equalization: true } }; var human = new Human(humanConfig); human.env["perfadd"] = false; diff --git a/demo/typescript/index.ts b/demo/typescript/index.ts index be0126ad..494f3acf 100644 --- a/demo/typescript/index.ts +++ b/demo/typescript/index.ts @@ -11,8 +11,8 @@ import { Human } from '../../dist/human.esm.js'; // equivalent of @vladmandic/Hu const humanConfig = { // user configuration for human, used to fine-tune behavior modelBasePath: '../../models', - filter: { equalization: false }, - // backend: 'webgpu', + filter: { equalization: true }, + // backend: 'webgpu' as 'webgpu, // async: true, // face: { enabled: false, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } }, // body: { enabled: false }, diff --git a/src/face/blazeface.ts b/src/face/blazeface.ts index 35c48d71..40d67951 100644 --- a/src/face/blazeface.ts +++ b/src/face/blazeface.ts @@ -13,7 +13,6 @@ import type { Point } from '../result'; const keypointsCount = 6; let model: GraphModel | null; -let anchorsData: [number, number][] = []; let anchors: Tensor | null = null; let inputSize = 0; @@ -27,9 +26,7 @@ export async function load(config: Config): Promise { else if (config.debug) log('load model:', model['modelUrl']); } else if (config.debug) log('cached model:', model['modelUrl']); inputSize = model.inputs[0].shape ? model.inputs[0].shape[2] : 0; - if (inputSize === -1) inputSize = 64; - anchorsData = util.generateAnchors(inputSize); - anchors = tf.tensor2d(anchorsData); + anchors = tf.tensor2d(util.generateAnchors(inputSize)); return model; } @@ -73,7 +70,6 @@ export async function getBoxes(inputImage: Tensor, config: Config) { t.logits = tf.slice(t.batch, [0, 0], [-1, 1]); t.sigmoid = tf.sigmoid(t.logits); t.scores = tf.squeeze(t.sigmoid); - t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0)); const nms = await t.nms.array() as number[]; const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = []; @@ -86,12 +82,11 @@ export async function getBoxes(inputImage: Tensor, config: Config) { b.slice = tf.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]); b.squeeze = tf.squeeze(b.slice); b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]); - b.startPoint = tf.slice(b.bbox, [0, 0], [-1, 2]); - b.endPoint = tf.slice(b.bbox, [0, 2], [-1, 2]); + const points = await b.bbox.data(); boxes.push({ box: { - startPoint: (await b.startPoint.data()) as unknown as Point, - endPoint: (await b.endPoint.data()) as unknown as Point, + startPoint: [points[0], points[1]] as Point, + endPoint: [points[2], points[3]] as Point, }, landmarks: (await b.landmarks.array()) as Point[], confidence, diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index 789aa545..44ac7d25 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -41,7 +41,10 @@ export async function predict(input: Tensor, config: Config): Promise (box ? [ ] : [0, 0, 0, 0]); export const scaleBoxCoordinates = (box, factor) => { - const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]]; - const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]]; + const startPoint: Point = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]]; + const endPoint: Point = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]]; return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence }; }; diff --git a/src/image/image.ts b/src/image/image.ts index 2556f6cf..997f3d30 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -262,7 +262,7 @@ const checksum = async (input: Tensor): Promise => { // use tf sum or js export async function skip(config, input: Tensor) { let skipFrame = false; - if (config.cacheSensitivity === 0) return skipFrame; + if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis /* const checkSum = await checksum(input);