ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add extra face rotation prior to mesh

pull/356/head
Vladimir Mandic 2021-11-16 13:07:44 -05:00
parent 6a6f14f658
commit 54b492b987
10 changed files with 36 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -9,8 +9,9 @@
## Changelog
### **HEAD -> main** 2021/11/14 mandic00@live.com
### **release 2.5.2** 2021/11/15 mandic00@live.com
- improve error handling
### **2.5.2** 2021/11/14 mandic00@live.com

9
TODO.md
View File

@ -6,9 +6,9 @@
### Exploring
- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
- Histogram Equalization: Regular, Adaptive, Contrast Limited, CLAHE
- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
- Optical flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
- Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE
- TFLite models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
- Body segmentation: `robust-video-matting`
- TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
@ -19,6 +19,7 @@
#### WebGPU
Experimental support only until support is officially added in Chromium
- Performance issues:
<https://github.com/tensorflow/tfjs/issues/5689>
@ -39,7 +40,7 @@ MoveNet MultiPose model does not work with WASM backend due to missing F32 broad
<br><hr><br>
## Pending release notes:
## Pending Release Notes
New:
- new demo `demos/faceid` that utilizes multiple algorithm to validate input before triggering face recognition

7
demo/faceid/README.md
View File

@ -32,3 +32,10 @@ designed to serve as a quick check when used together with other indicators:
- Checks if input has obvious artifacts due to recording (e.g. playing back phone recording of a face)
- Configuration: `human.config.face.liveness`.enabled
- Result: `human.result.face[0].live` as score
### Models
**FaceID** is compatible with
- `faceres.json` (default) perfoms combined age/gender/descriptor analysis
- `faceres-deep.json` higher resolution variation of `faceres`
- `mobilefacenet` alternative model for face descriptor analysis

15
demo/facematch/facematch.js
View File

@ -19,10 +19,12 @@ const userConfig = {
filter: {
enabled: true,
equalization: true,
width: 0,
},
face: {
enabled: true,
detector: { rotation: true, return: true, maxDetected: 50 },
// detector: { rotation: false, return: true, maxDetected: 50, iouThreshold: 0.206, minConfidence: 0.122 },
detector: { return: true, rotation: true, maxDetected: 50, iouThreshold: 0.01, minConfidence: 0.2 },
mesh: { enabled: true },
iris: { enabled: false },
emotion: { enabled: true },
@ -138,7 +140,8 @@ async function SelectFaceCanvas(face) {
async function AddFaceCanvas(index, res, fileName) {
all[index] = res.face;
for (const i in res.face) {
if (res.face[i].mesh.length === 0 || !res.face[i].tensor) continue; // did not get valid results
if (!res.face[i].tensor) continue; // did not get valid results
if ((res.face[i].faceScore || 0) < human.config.face.detector.minConfidence) continue; // face analysis score too low
all[index][i].fileName = fileName;
const canvas = document.createElement('canvas');
canvas.tag = { sample: index, face: i, source: fileName };
@ -177,9 +180,9 @@ async function AddImageElement(index, image, length) {
return new Promise((resolve) => {
const img = new Image(128, 128);
img.onload = () => { // must wait until image is loaded
document.getElementById('images').appendChild(img); // and finally we can add it
human.detect(img, userConfig).then((res) => {
AddFaceCanvas(index, res, image); // then wait until image is analyzed
document.getElementById('images').appendChild(img); // and finally we can add it
resolve(true);
});
};
@ -236,12 +239,8 @@ async function main() {
log('Discovered images:', images);
}
// images = ['/samples/in/solvay1927.jpg'];
// images = ['/samples/in/person-lexi.jpg', '/samples/in/person-carolina.jpg', '/samples/in/solvay1927.jpg'];
// download and analyze all images
// const promises = [];
// for (let i = 0; i < images.length; i++) promises.push(AddImageElement(i, images[i], images.length));
// await Promise.all(promises);
const t0 = human.now();
for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length);
const t1 = human.now();

2
demo/typescript/index.js
View File

@ -8,7 +8,7 @@
import { Human } from "../../dist/human.esm.js";
var humanConfig = {
modelBasePath: "../../models",
filter: { equalization: false }
filter: { equalization: true }
};
var human = new Human(humanConfig);
human.env["perfadd"] = false;

4
demo/typescript/index.ts
View File

@ -11,8 +11,8 @@ import { Human } from '../../dist/human.esm.js'; // equivalent of @vladmandic/Hu
const humanConfig = { // user configuration for human, used to fine-tune behavior
modelBasePath: '../../models',
filter: { equalization: false },
// backend: 'webgpu',
filter: { equalization: true },
// backend: 'webgpu' as 'webgpu,
// async: true,
// face: { enabled: false, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
// body: { enabled: false },

13
src/face/blazeface.ts
View File

@ -13,7 +13,6 @@ import type { Point } from '../result';
const keypointsCount = 6;
let model: GraphModel | null;
let anchorsData: [number, number][] = [];
let anchors: Tensor | null = null;
let inputSize = 0;
@ -27,9 +26,7 @@ export async function load(config: Config): Promise<GraphModel> {
else if (config.debug) log('load model:', model['modelUrl']);
} else if (config.debug) log('cached model:', model['modelUrl']);
inputSize = model.inputs[0].shape ? model.inputs[0].shape[2] : 0;
if (inputSize === -1) inputSize = 64;
anchorsData = util.generateAnchors(inputSize);
anchors = tf.tensor2d(anchorsData);
anchors = tf.tensor2d(util.generateAnchors(inputSize));
return model;
}
@ -73,7 +70,6 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf.sigmoid(t.logits);
t.scores = tf.squeeze(t.sigmoid);
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
const nms = await t.nms.array() as number[];
const boxes: Array<{ box: { startPoint: Point, endPoint: Point }, landmarks: Point[], confidence: number }> = [];
@ -86,12 +82,11 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
b.slice = tf.slice(t.batch, [nms[i], keypointsCount - 1], [1, -1]);
b.squeeze = tf.squeeze(b.slice);
b.landmarks = tf.reshape(b.squeeze, [keypointsCount, -1]);
b.startPoint = tf.slice(b.bbox, [0, 0], [-1, 2]);
b.endPoint = tf.slice(b.bbox, [0, 2], [-1, 2]);
const points = await b.bbox.data();
boxes.push({
box: {
startPoint: (await b.startPoint.data()) as unknown as Point,
endPoint: (await b.endPoint.data()) as unknown as Point,
startPoint: [points[0], points[1]] as Point,
endPoint: [points[2], points[3]] as Point,
},
landmarks: (await b.landmarks.array()) as Point[],
confidence,

7
src/face/facemesh.ts
View File

@ -41,7 +41,10 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
landmarks: possible.landmarks,
confidence: possible.confidence,
};
boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(config.face.detector?.cropFactor || 1.6))));
const boxScaled = util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor);
const boxEnlarged = util.enlargeBox(boxScaled, Math.sqrt(config.face.detector?.cropFactor || 1.6));
const boxSquared = util.squarifyBox(boxEnlarged);
boxCache.push(boxSquared);
}
skipped = 0;
} else {
@ -67,7 +70,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
};
// optional rotation correction based on detector data only if mesh is disabled otherwise perform it later when we have more accurate mesh data. if no rotation correction this function performs crop
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(!config.face.mesh?.enabled && config.face.detector?.rotation, box, input, config.face.mesh?.enabled ? inputSize : blazeface.size());
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(config.face.detector?.rotation, box, input, config.face.mesh?.enabled ? inputSize : blazeface.size());
if (config?.filter?.equalization) {
const equilized = await histogramEqualization(face.tensor as Tensor);
tf.dispose(face.tensor);

4
src/face/facemeshutil.ts
View File

@ -31,8 +31,8 @@ export const getRawBox = (box, input): Box => (box ? [
] : [0, 0, 0, 0]);
export const scaleBoxCoordinates = (box, factor) => {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const startPoint: Point = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint: Point = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence };
};

2
src/image/image.ts
View File

@ -262,7 +262,7 @@ const checksum = async (input: Tensor): Promise<number> => { // use tf sum or js
export async function skip(config, input: Tensor) {
let skipFrame = false;
if (config.cacheSensitivity === 0) return skipFrame;
if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis
/*
const checkSum = await checksum(input);