mirror of https://github.com/vladmandic/human
improve box rescaling for all modules
Vladimir Mandic
parent
e45509d163
commit
1ddbdcdd2e
|
|
@ -9,11 +9,9 @@
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/10/31 mandic00@live.com
|
||||
|
||||
|
||||
### **origin/main** 2021/10/31 mandic00@live.com
|
||||
### **HEAD -> main** 2021/11/02 mandic00@live.com
|
||||
|
||||
- refactor predict with execute
|
||||
- patch tfjs type defs
|
||||
- start 2.5 major version
|
||||
- build and docs cleanup
|
||||
|
|
|
|||
|
|
@ -9,12 +9,7 @@ import Human from "../../dist/human.esm.js";
|
|||
var config = {
|
||||
modelBasePath: "../../models",
|
||||
backend: "humangl",
|
||||
async: true,
|
||||
face: { enabled: true },
|
||||
body: { enabled: true },
|
||||
hand: { enabled: true },
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true }
|
||||
async: true
|
||||
};
|
||||
var human = new Human(config);
|
||||
human.env.perfadd = false;
|
||||
|
|
@ -39,7 +34,7 @@ var perf = (msg) => {
|
|||
};
|
||||
async function webCam() {
|
||||
status("starting webcam...");
|
||||
const options = { audio: false, video: { facingMode: "user", resizeMode: "none", width: { ideal: document.body.clientWidth } } };
|
||||
const options = { audio: false, video: { facingMode: "user", resizeMode: "crop-and-scale", width: { ideal: document.body.clientWidth } } };
|
||||
const stream = await navigator.mediaDevices.getUserMedia(options);
|
||||
const ready = new Promise((resolve) => {
|
||||
dom.video.onloadeddata = () => resolve(true);
|
||||
|
|
|
|||
|
|
@ -15,11 +15,11 @@ const config = {
|
|||
modelBasePath: '../../models',
|
||||
backend: 'humangl',
|
||||
async: true,
|
||||
face: { enabled: true },
|
||||
body: { enabled: true },
|
||||
hand: { enabled: true },
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
// face: { enabled: true, detector: { rotation: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } },
|
||||
// body: { enabled: false },
|
||||
// hand: { enabled: false },
|
||||
// object: { enabled: false },
|
||||
// gesture: { enabled: true },
|
||||
};
|
||||
|
||||
const human = new Human(config);
|
||||
|
|
@ -50,7 +50,7 @@ const perf = (msg) => {
|
|||
|
||||
async function webCam() {
|
||||
status('starting webcam...');
|
||||
const options = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };
|
||||
const options = { audio: false, video: { facingMode: 'user', resizeMode: 'crop-and-scale', width: { ideal: document.body.clientWidth } } };
|
||||
const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);
|
||||
const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });
|
||||
dom.video.srcObject = stream;
|
||||
|
|
|
|||
|
|
@ -18,48 +18,45 @@ import type { FaceResult, Point } from '../result';
|
|||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number, faceConfidence?: number | undefined };
|
||||
type BoxCache = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
|
||||
let boxCache: Array<BoxCache> = [];
|
||||
let model: GraphModel | null = null;
|
||||
let inputSize = 0;
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
let lastTime = 0;
|
||||
let detectedFaces = 0;
|
||||
const enlargeFact = 1.6;
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
|
||||
// reset cached boxes
|
||||
|
||||
const skipTime = (config.face.detector?.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.face.detector?.skipFrames || 0);
|
||||
if (!config.skipAllowed || !skipTime || !skipFrame || detectedFaces === 0) {
|
||||
const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
|
||||
if (!config.skipAllowed || !skipTime || !skipFrame || boxCache.length === 0) {
|
||||
const possibleBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
|
||||
lastTime = now();
|
||||
boxCache = []; // empty cache
|
||||
for (const possible of newBoxes.boxes) { // extract data from detector
|
||||
const startPoint = await possible.box.startPoint.data() as unknown as Point;
|
||||
const endPoint = await possible.box.endPoint.data() as unknown as Point;
|
||||
const landmarks = await possible.landmarks.array() as Array<Point>;
|
||||
boxCache.push({ startPoint, endPoint, landmarks, confidence: possible.confidence });
|
||||
}
|
||||
newBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
for (let i = 0; i < boxCache.length; i++) { // enlarge and squarify detected boxes
|
||||
const scaledBox = util.scaleBoxCoordinates({ startPoint: boxCache[i].startPoint, endPoint: boxCache[i].endPoint }, newBoxes.scaleFactor);
|
||||
const enlargedBox = util.enlargeBox(scaledBox);
|
||||
const squarifiedBox = util.squarifyBox(enlargedBox);
|
||||
boxCache[i] = { ...squarifiedBox, confidence: boxCache[i].confidence, landmarks: boxCache[i].landmarks };
|
||||
for (const possible of possibleBoxes.boxes) { // extract data from detector
|
||||
const box: BoxCache = {
|
||||
startPoint: await possible.box.startPoint.data() as unknown as Point,
|
||||
endPoint: await possible.box.endPoint.data() as unknown as Point,
|
||||
landmarks: await possible.landmarks.array() as Array<Point>,
|
||||
confidence: possible.confidence,
|
||||
};
|
||||
boxCache.push(util.squarifyBox(util.enlargeBox(util.scaleBoxCoordinates(box, possibleBoxes.scaleFactor), Math.sqrt(enlargeFact))));
|
||||
}
|
||||
possibleBoxes.boxes.forEach((prediction) => tf.dispose([prediction.box.startPoint, prediction.box.endPoint, prediction.landmarks]));
|
||||
skipped = 0;
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
|
||||
const faces: Array<FaceResult> = [];
|
||||
const newBoxes: Array<BoxCache> = [];
|
||||
const newCache: Array<BoxCache> = [];
|
||||
let id = 0;
|
||||
for (let box of boxCache) {
|
||||
for (let i = 0; i < boxCache.length; i++) {
|
||||
let box = boxCache[i];
|
||||
let angle = 0;
|
||||
let rotationMatrix;
|
||||
const face: FaceResult = {
|
||||
const face: FaceResult = { // init face result
|
||||
id: id++,
|
||||
mesh: [],
|
||||
meshRaw: [],
|
||||
|
|
@ -74,16 +71,15 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
|
|||
if (config.face.detector?.rotation && config.face.mesh?.enabled && env.kernels.includes('rotatewithoffset')) {
|
||||
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(box, input, inputSize);
|
||||
} else {
|
||||
rotationMatrix = util.IDENTITY_MATRIX;
|
||||
const cut = util.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, input, config.face.mesh?.enabled ? [inputSize, inputSize] : [blazeface.size(), blazeface.size()]);
|
||||
face.tensor = tf.div(cut, 255);
|
||||
tf.dispose(cut);
|
||||
rotationMatrix = util.fixedRotationMatrix;
|
||||
face.tensor = util.cutBoxFromImageAndResize(box, input, config.face.mesh?.enabled ? [inputSize, inputSize] : [blazeface.size(), blazeface.size()]);
|
||||
}
|
||||
face.boxScore = Math.round(100 * box.confidence) / 100;
|
||||
if (!config.face.mesh?.enabled) { // mesh not enabled, return resuts from detector only
|
||||
face.box = util.getClampedBox(box, input);
|
||||
face.boxRaw = util.getRawBox(box, input);
|
||||
face.score = Math.round(100 * box.confidence || 0) / 100;
|
||||
face.boxScore = Math.round(100 * box.confidence || 0) / 100;
|
||||
face.score = face.boxScore;
|
||||
face.mesh = box.landmarks.map((pt) => [
|
||||
((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()),
|
||||
((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()),
|
||||
|
|
@ -94,37 +90,36 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
|
|||
if (config.debug) log('face mesh detection requested, but model is not loaded');
|
||||
} else { // mesh enabled
|
||||
const [contours, confidence, contourCoords] = model.execute(face.tensor as Tensor) as Array<Tensor>; // first returned tensor represents facial contours which are already included in the coordinates.
|
||||
tf.dispose(contours);
|
||||
const faceConfidence = (await confidence.data())[0] as number;
|
||||
tf.dispose(confidence);
|
||||
const faceConfidence = await confidence.data();
|
||||
face.faceScore = Math.round(100 * faceConfidence[0]) / 100;
|
||||
const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
|
||||
let rawCoords = await coordsReshaped.array();
|
||||
tf.dispose(contourCoords);
|
||||
tf.dispose(coordsReshaped);
|
||||
if (faceConfidence < (config.face.detector?.minConfidence || 1)) {
|
||||
box.confidence = faceConfidence; // reset confidence of cached box
|
||||
tf.dispose([contourCoords, coordsReshaped, confidence, contours]);
|
||||
if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh
|
||||
box.confidence = face.faceScore; // reset confidence of cached box
|
||||
} else {
|
||||
if (config.face.iris?.enabled) rawCoords = await iris.augmentIris(rawCoords, face.tensor, config, inputSize); // augment results with iris
|
||||
face.mesh = util.transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize); // get processed mesh
|
||||
face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]);
|
||||
box = { ...util.enlargeBox(util.calculateLandmarksBoundingBox(face.mesh), 1.5), confidence: box.confidence }; // redefine box with mesh calculated one
|
||||
for (const key of Object.keys(coords.meshAnnotations)) face.annotations[key] = coords.meshAnnotations[key].map((index) => face.mesh[index]); // add annotations
|
||||
if (config.face.detector?.rotation && config.face.mesh.enabled && config.face.description?.enabled && env.kernels.includes('rotatewithoffset')) { // do rotation one more time with mesh keypoints if we want to return perfect image
|
||||
tf.dispose(face.tensor); // dispose so we can overwrite original face
|
||||
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(box, input, inputSize);
|
||||
}
|
||||
box = util.squarifyBox(util.enlargeBox(util.calculateLandmarksBoundingBox(face.mesh), enlargeFact)); // redefine box with mesh calculated one
|
||||
face.box = util.getClampedBox(box, input); // update detected box with box around the face mesh
|
||||
face.boxRaw = util.getRawBox(box, input);
|
||||
face.score = Math.round(100 * faceConfidence || 100 * box.confidence || 0) / 100;
|
||||
face.faceScore = Math.round(100 * faceConfidence) / 100;
|
||||
box = { ...util.squarifyBox(box), confidence: box.confidence, faceConfidence }; // updated stored cache values
|
||||
face.score = face.faceScore;
|
||||
newCache.push(box);
|
||||
|
||||
// other modules prefer wider crop for a face so we dispose it and do it again
|
||||
/*
|
||||
tf.dispose(face.tensor);
|
||||
face.tensor = config.face.detector?.rotation && config.face.mesh?.enabled && env.kernels.includes('rotatewithoffset')
|
||||
? face.tensor = util.correctFaceRotation(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, inputSize)[2]
|
||||
: face.tensor = util.cutBoxFromImageAndResize(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, [inputSize, inputSize]);
|
||||
*/
|
||||
}
|
||||
}
|
||||
faces.push(face);
|
||||
newBoxes.push(box);
|
||||
}
|
||||
if (config.face.mesh?.enabled) boxCache = newBoxes.filter((a) => a.confidence > (config.face.detector?.minConfidence || 0)); // remove cache entries for detected boxes on low confidence
|
||||
detectedFaces = faces.length;
|
||||
boxCache = [...newCache]; // reset cache
|
||||
return faces;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,36 +32,39 @@ export const getRawBox = (box, input): Box => (box ? [
|
|||
export const scaleBoxCoordinates = (box, factor) => {
|
||||
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
|
||||
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
|
||||
return { startPoint, endPoint };
|
||||
return { startPoint, endPoint, landmarks: box.landmarks, confidence: box.confidence };
|
||||
};
|
||||
|
||||
export const cutBoxFromImageAndResize = (box, image, cropSize) => {
|
||||
const h = image.shape[1];
|
||||
const w = image.shape[2];
|
||||
return tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize);
|
||||
const crop = tf.image.cropAndResize(image, [[box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.endPoint[0] / w]], [0], cropSize);
|
||||
const norm = tf.div(crop, 255);
|
||||
tf.dispose(crop);
|
||||
return norm;
|
||||
};
|
||||
|
||||
export const enlargeBox = (box, factor = 1.5) => {
|
||||
export const enlargeBox = (box, factor) => {
|
||||
const center = getBoxCenter(box);
|
||||
const size = getBoxSize(box);
|
||||
const halfSize: [number, number] = [factor * size[0] / 2, factor * size[1] / 2];
|
||||
return { startPoint: [center[0] - halfSize[0], center[1] - halfSize[1]] as Point, endPoint: [center[0] + halfSize[0], center[1] + halfSize[1]] as Point, landmarks: box.landmarks };
|
||||
return { startPoint: [center[0] - halfSize[0], center[1] - halfSize[1]] as Point, endPoint: [center[0] + halfSize[0], center[1] + halfSize[1]] as Point, landmarks: box.landmarks, confidence: box.confidence };
|
||||
};
|
||||
|
||||
export const squarifyBox = (box) => {
|
||||
const centers = getBoxCenter(box);
|
||||
const size = getBoxSize(box);
|
||||
const halfSize = Math.max(...size) / 2;
|
||||
return { startPoint: [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)] as Point, endPoint: [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)] as Point, landmarks: box.landmarks };
|
||||
return { startPoint: [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)] as Point, endPoint: [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)] as Point, landmarks: box.landmarks, confidence: box.confidence };
|
||||
};
|
||||
|
||||
export const calculateLandmarksBoundingBox = (landmarks) => {
|
||||
const xs = landmarks.map((d) => d[0]);
|
||||
const ys = landmarks.map((d) => d[1]);
|
||||
return { startPoint: [Math.min(...xs), Math.min(...ys)], endPoint: [Math.max(...xs), Math.max(...ys)], landmarks };
|
||||
return { startPoint: [Math.min(...xs), Math.min(...ys)] as Point, endPoint: [Math.max(...xs), Math.max(...ys)] as Point, landmarks };
|
||||
};
|
||||
|
||||
export const IDENTITY_MATRIX = [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
|
||||
export const fixedRotationMatrix = [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
|
||||
|
||||
export const normalizeRadians = (angle) => angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI));
|
||||
|
||||
|
|
@ -71,7 +74,7 @@ export const radToDegrees = (rad) => rad * 180 / Math.PI;
|
|||
|
||||
export const buildTranslationMatrix = (x, y) => [[1, 0, x], [0, 1, y], [0, 0, 1]];
|
||||
|
||||
export const dot = (v1, v2) => {
|
||||
export const dot = (v1: number[], v2: number[]) => {
|
||||
let product = 0;
|
||||
for (let i = 0; i < v1.length; i++) product += v1[i] * v2[i];
|
||||
return product;
|
||||
|
|
@ -133,16 +136,17 @@ export function generateAnchors(inputSize) {
|
|||
return anchors;
|
||||
}
|
||||
|
||||
export function transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize) {
|
||||
const boxSize = getBoxSize({ startPoint: box.startPoint, endPoint: box.endPoint });
|
||||
const coordsScaled = rawCoords.map((coord) => ([
|
||||
export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputSize) {
|
||||
const boxSize = getBoxSize(box);
|
||||
const coordsScaled = coordsRaw.map((coord) => ([ // scaled around zero-point
|
||||
boxSize[0] / inputSize * (coord[0] - inputSize / 2),
|
||||
boxSize[1] / inputSize * (coord[1] - inputSize / 2),
|
||||
coord[2] || 0,
|
||||
]));
|
||||
const coordsRotationMatrix = (angle !== 0) ? buildRotationMatrix(angle, [0, 0]) : IDENTITY_MATRIX;
|
||||
const coordsRotated = (angle !== 0) ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled;
|
||||
const inverseRotationMatrix = (angle !== 0) ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
|
||||
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
|
||||
const coordsRotationMatrix = largeAngle ? buildRotationMatrix(angle, [0, 0]) : fixedRotationMatrix;
|
||||
const coordsRotated = largeAngle ? coordsScaled.map((coord) => ([...rotatePoint(coord, coordsRotationMatrix), coord[2]])) : coordsScaled;
|
||||
const inverseRotationMatrix = largeAngle ? invertTransformMatrix(rotationMatrix) : fixedRotationMatrix;
|
||||
const boxCenter = [...getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1];
|
||||
return coordsRotated.map((coord) => ([
|
||||
Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
|
||||
|
|
@ -154,13 +158,19 @@ export function transformRawCoords(rawCoords, box, angle, rotationMatrix, inputS
|
|||
export function correctFaceRotation(box, input, inputSize) {
|
||||
const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count) ? coords.meshLandmarks.symmetryLine : coords.blazeFaceLandmarks.symmetryLine;
|
||||
const angle: number = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]);
|
||||
const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
|
||||
const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
|
||||
const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
|
||||
const rotationMatrix = buildRotationMatrix(-angle, faceCenter);
|
||||
const cut = cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotated, [inputSize, inputSize]);
|
||||
const face = tf.div(cut, 255);
|
||||
tf.dispose(cut);
|
||||
tf.dispose(rotated);
|
||||
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
|
||||
let rotationMatrix;
|
||||
let face;
|
||||
if (largeAngle) {
|
||||
const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
|
||||
const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
|
||||
const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
|
||||
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
|
||||
face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]);
|
||||
tf.dispose(rotated);
|
||||
} else {
|
||||
rotationMatrix = fixedRotationMatrix;
|
||||
face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]);
|
||||
}
|
||||
return [angle, rotationMatrix, face];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,13 +43,14 @@ export function enhance(input): Tensor {
|
|||
const tensor = input.image || input.tensor || input;
|
||||
if (!(tensor instanceof tf.Tensor)) return null;
|
||||
// do a tight crop of image and resize it to fit the model
|
||||
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
// const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test
|
||||
if (!model?.inputs[0].shape) return null; // model has no shape so no point continuing
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
/*
|
||||
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
const crop = (tensor.shape.length === 3)
|
||||
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
|
||||
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
|
||||
*/
|
||||
/*
|
||||
// just resize to fit the embedding model instead of cropping
|
||||
const crop = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
|
|
|
|||
|
|
@ -43,7 +43,11 @@ export async function predict(image: Tensor, config: Config, idx, count) {
|
|||
return new Promise(async (resolve) => {
|
||||
const obj: Array<{ score: number, emotion: string }> = [];
|
||||
if (config.face.emotion?.enabled) {
|
||||
const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false);
|
||||
const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0;
|
||||
const resize = tf.image.resizeBilinear(image, [inputSize, inputSize], false);
|
||||
// const box = [[0.15, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
// const resize = tf.image.cropAndResize(image, box, [0], [inputSize, inputSize]);
|
||||
|
||||
const [red, green, blue] = tf.split(resize, 3, 3);
|
||||
tf.dispose(resize);
|
||||
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
|
||||
|
|
|
|||
|
|
@ -100,11 +100,14 @@ export const iris = (res): GestureResult[] => {
|
|||
gestures.push({ iris: i, gesture: 'facing center' });
|
||||
}
|
||||
|
||||
const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].box[2];
|
||||
const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].box[2];
|
||||
const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].box[2];
|
||||
if (leftIrisCenterX > 0.06 || rightIrisCenterX > 0.06) center = false;
|
||||
if (leftIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking right' });
|
||||
if (rightIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking left' });
|
||||
if (leftIrisCenterX > rightIrisCenterX) { // check eye with bigger offset
|
||||
if (leftIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking right' });
|
||||
} else {
|
||||
if (rightIrisCenterX > 0.05) gestures.push({ iris: i, gesture: 'looking left' });
|
||||
}
|
||||
|
||||
const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].box[3];
|
||||
const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].box[3];
|
||||
|
|
|
|||
|
|
@ -188,12 +188,19 @@ async function test(Human, inputConfig) {
|
|||
else log('state', 'passed: warmup none result match');
|
||||
config.warmup = 'face';
|
||||
res = await testWarmup(human, 'default');
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 8) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
else log('state', 'passed: warmup face result match');
|
||||
config.warmup = 'body';
|
||||
res = await testWarmup(human, 'default');
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 6) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
else log('state', 'passed: warmup body result match');
|
||||
log('state', 'details:', {
|
||||
face: { boxScore: res.face[0].boxScore, faceScore: res.face[0].faceScore, age: res.face[0].age, gender: res.face[0].gender, genderScore: res.face[0].genderScore },
|
||||
emotion: res.face[0].emotion,
|
||||
body: { score: res.body[0].score, keypoints: res.body[0].keypoints.length },
|
||||
hand: { boxScore: res.hand[0].boxScore, fingerScore: res.hand[0].fingerScore, keypoints: res.hand[0].keypoints.length },
|
||||
gestures: res.gesture,
|
||||
});
|
||||
|
||||
// test default config async
|
||||
log('info', 'test default');
|
||||
|
|
@ -201,8 +208,8 @@ async function test(Human, inputConfig) {
|
|||
config.async = true;
|
||||
config.cacheSensitivity = 0;
|
||||
res = await testDetect(human, 'samples/in/ai-body.jpg', 'default');
|
||||
if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default result face mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
else log('state', 'passed: default result face match');
|
||||
if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default result face mismatch', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore);
|
||||
else log('state', 'passed: default result face match', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore);
|
||||
|
||||
// test default config sync
|
||||
log('info', 'test sync');
|
||||
|
|
@ -210,8 +217,8 @@ async function test(Human, inputConfig) {
|
|||
config.async = false;
|
||||
config.cacheSensitivity = 0;
|
||||
res = await testDetect(human, 'samples/in/ai-body.jpg', 'default');
|
||||
if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default sync', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
else log('state', 'passed: default sync');
|
||||
if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default sync', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore);
|
||||
else log('state', 'passed: default sync', res?.face?.length, res?.face[0].gender, res?.face[0].genderScore);
|
||||
|
||||
// test image processing
|
||||
const img1 = await human.image(null);
|
||||
|
|
@ -240,7 +247,7 @@ async function test(Human, inputConfig) {
|
|||
res1 = human.similarity(desc1, desc1);
|
||||
res2 = human.similarity(desc1, desc2);
|
||||
res3 = human.similarity(desc1, desc3);
|
||||
if (res1 < 1 || res2 < 0.55 || res3 < 0.5) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
|
||||
if (res1 < 1 || res2 < 0.50 || res3 < 0.50) log('error', 'failed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
|
||||
else log('state', 'passed: face similarity', { similarity: [res1, res2, res3], descriptors: [desc1?.length, desc2?.length, desc3?.length] });
|
||||
|
||||
// test face matching
|
||||
|
|
@ -271,7 +278,7 @@ async function test(Human, inputConfig) {
|
|||
config.body = { minConfidence: 0.0001 };
|
||||
config.hand = { minConfidence: 0.0001 };
|
||||
res = await testDetect(human, 'samples/in/ai-body.jpg', 'default');
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 9) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
|
||||
else log('state', 'passed: sensitive result match');
|
||||
|
||||
// test sensitive details face
|
||||
|
|
@ -280,7 +287,7 @@ async function test(Human, inputConfig) {
|
|||
log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.embedding?.length, face?.rotation?.matrix?.length);
|
||||
} else log('state', 'passed: sensitive face result match');
|
||||
if (!face || face?.emotion?.length < 3) log('error', 'failed: sensitive face emotion result mismatch', face?.emotion.length);
|
||||
else log('state', 'passed: sensitive face emotion result mismatch', face?.emotion.length);
|
||||
else log('state', 'passed: sensitive face emotion result', face?.emotion.length);
|
||||
|
||||
// test sensitive details body
|
||||
const body = res && res.body ? res.body[0] : null;
|
||||
|
|
|
|||
Loading…
Reference in New Issue