ambiser
/
face-api
mirror of https://github.com/vladmandic/face-api.git
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix face angles (yaw, pitch, & roll) accuracy (#130) 

Previouly derived aforementioned angles correctly seemed inaccurate and somewhat unusable (given their output was in radians). This update uses the a person's mesh positions, and chooses specific points for accurate results. It also adds directionality of the movements (_e.g. pitching head backwards is a negative result, as is rolling head to the left).

The webcam.js file has also been updated to showcase the correct output in degrees (reducing potential user confusion)

Comitter: Sohaib Ahmed <sohaibi.ahmed@icloud.com>

Co-authored-by: Sophia Glisch <sophiaglisch@Sophias-MacBook-Pro.local>
pull/131/head
Sohaib Ahmed 2022-10-18 07:09:35 -04:00 committed by GitHub
parent cd2c553737
commit 7e5a1289ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 102 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
demo/webcam.js
View File

@ -53,12 +53,12 @@ function drawFaces(canvas, data, fps) {
ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 59);
ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 41);
ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 23);
ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 5);
ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 5);
ctx.fillStyle = 'lightblue';
ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 60);
ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 42);
ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 24);
ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 6);
ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 6);
// draw face points for each face
ctx.globalAlpha = 0.8;
ctx.fillStyle = 'lightblue';

134
src/factories/WithFaceLandmarks.ts
View File

@ -5,64 +5,130 @@ import { isWithFaceDetection, WithFaceDetection } from './WithFaceDetection';
export type WithFaceLandmarks<
TSource extends WithFaceDetection<{}>,
TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 > = TSource & {
landmarks: TFaceLandmarks,
unshiftedLandmarks: TFaceLandmarks,
alignedRect: FaceDetection,
angle: { roll: number | undefined, pitch: number | undefined, yaw: number | undefined },
}
TFaceLandmarks extends FaceLandmarks = FaceLandmarks68
> = TSource & {
landmarks: TFaceLandmarks;
unshiftedLandmarks: TFaceLandmarks;
alignedRect: FaceDetection;
angle: {
roll: number | undefined;
pitch: number | undefined;
yaw: number | undefined;
};
};
export function isWithFaceLandmarks(obj: any): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
return isWithFaceDetection(obj)
export function isWithFaceLandmarks(
obj: any,
): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
return (
isWithFaceDetection(obj)
// eslint-disable-next-line dot-notation
&& obj['landmarks'] instanceof FaceLandmarks
// eslint-disable-next-line dot-notation
&& obj['unshiftedLandmarks'] instanceof FaceLandmarks
// eslint-disable-next-line dot-notation
&& obj['alignedRect'] instanceof FaceDetection;
&& obj['alignedRect'] instanceof FaceDetection
);
}
function calculateFaceAngle(mesh) {
// returns the angle in the plane (in radians) between the positive x-axis and the ray from (0,0) to the point (x,y)
const radians = (a1, a2, b1, b2) => (Math.atan2(b2 - a2, b1 - a1) % Math.PI);
// convert radians to degrees
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const degrees = (theta) => (theta * 180) / Math.PI;
/*
AUTHORED BY: SOHAIB AHMED
https://github.com/TheSohaibAhmed/
*/
const angle = { roll: <number | undefined>undefined, pitch: <number | undefined>undefined, yaw: <number | undefined>undefined };
// Helper to convert radians to degrees
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const degrees = (radians) => (radians * 180) / Math.PI;
const angle = {
roll: <number | undefined>undefined,
pitch: <number | undefined>undefined,
yaw: <number | undefined>undefined,
};
if (!mesh || !mesh._positions || mesh._positions.length !== 68) return angle;
const pt = mesh._positions;
// values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees
// value of 0 means center
function calcLengthBetweenTwoPoints(a, b) {
return Math.sqrt((a._x - b._x) ** 2 + (a._y - b._y) ** 2);
}
const calcYaw = (leftPoint, midPoint, rightPoint) => {
// Calc x-distance from left side of the face ("ear") to facial midpoint ("nose")
const leftToMidpoint = Math.floor(leftPoint._x - midPoint._x);
// roll is face lean from left to right
// comparing x,y of outside corners of leftEye and rightEye
angle.roll = -radians(pt[36]._x, pt[36]._y, pt[45]._x, pt[45]._y);
// Calc x-distance from facial midpoint ("nose") to the right side of the face ("ear")
const rightToMidpoint = Math.floor(midPoint._x - rightPoint._x);
// pitch is face turn from left right
// comparing x distance of top of nose to left and right edge of face
// precision is lacking since coordinates are not precise enough
angle.pitch = radians(0, Math.abs(pt[0]._x - pt[30]._x) / pt[30]._x, Math.PI, Math.abs(pt[16]._x - pt[30]._x) / pt[30]._x);
// Difference in distances coincidentally approximates to angles
const distanceApproximatesToAngle = leftToMidpoint - rightToMidpoint;
return distanceApproximatesToAngle;
};
// yaw is face move from up to down
// comparing size of the box around the face with top and bottom of detected landmarks
// silly hack, but this gives us face compression on y-axis
// e.g., tilting head up hides the forehead that doesn't have any landmarks so ratio drops
const bottom = pt.reduce((prev, cur) => (prev < cur._y ? prev : cur._y), +Infinity);
const top = pt.reduce((prev, cur) => (prev > cur._y ? prev : cur._y), -Infinity);
angle.yaw = Math.PI * (mesh._imgDims._height / (top - bottom) / 1.40 - 1);
const calcRoll = (lever, pivot) => {
// When rolling, the head seems to pivot from the nose/lips/chin area.
// So, we'll choose any two points from the facial midline, where the first point should be the pivot, and the other "lever"
// Plan/Execution: get the hypotenuse & opposite sides of a 90deg triangle ==> Calculate angle in radians
const hypotenuse = Math.hypot(pivot._x - lever._x, pivot._y - lever._y);
const opposite = pivot._y - lever._y;
const angleInRadians = Math.asin(opposite / hypotenuse);
const angleInDegrees = degrees(angleInRadians);
const normalizeAngle = Math.floor(90 - angleInDegrees);
// If lever more to the left of the pivot, then we're tilting left
// "-" is negative direction. "+", or absence of a sign is positive direction
const tiltDirection = pivot._x - lever._x < 0 ? -1 : 1;
const result = normalizeAngle * tiltDirection;
return result;
};
const calcPitch = (leftPoint, midPoint, rightPoint) => {
// Theory: While pitching, the nose is the most salient point --> That's what we'll use to make a trianle.
// The "base" is between point that don't move when we pitch our head (i.e. an imaginary line running ear to ear through the nose).
// Executuin: Get the opposite & adjacent lengths of the triangle from the ear's perspective. Use it to get angle.
const base = calcLengthBetweenTwoPoints(leftPoint, rightPoint);
// adjecent is base/2 technically.
const baseCoords = {
_x: (leftPoint._x + rightPoint._x) / 2,
_y: (leftPoint._y + rightPoint._y) / 2,
};
const midToBaseLength = calcLengthBetweenTwoPoints(midPoint, baseCoords);
const angleInRadians = Math.atan(midToBaseLength / base);
const angleInDegrees = Math.floor(degrees(angleInRadians));
// Account for directionality.
// pitch forwards (_i.e. tilting your head forwards) is positive (or no sign); backward is negative.
const direction = baseCoords._y - midPoint._y < 0 ? -1 : 1;
const result = angleInDegrees * direction;
return result;
};
angle.roll = calcRoll(pt[27], pt[66]);
angle.pitch = calcPitch(pt[14], pt[30], pt[2]);
angle.yaw = calcYaw(pt[14], pt[33], pt[2]);
return angle;
}
export function extendWithFaceLandmarks<TSource extends WithFaceDetection<{}>, TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 >(sourceObj: TSource, unshiftedLandmarks: TFaceLandmarks): WithFaceLandmarks<TSource, TFaceLandmarks> {
export function extendWithFaceLandmarks<
TSource extends WithFaceDetection<{}>,
TFaceLandmarks extends FaceLandmarks = FaceLandmarks68
>(
sourceObj: TSource,
unshiftedLandmarks: TFaceLandmarks,
): WithFaceLandmarks<TSource, TFaceLandmarks> {
const { box: shift } = sourceObj.detection;
const landmarks = unshiftedLandmarks.shiftBy<TFaceLandmarks>(shift.x, shift.y);
const landmarks = unshiftedLandmarks.shiftBy<TFaceLandmarks>(
shift.x,
shift.y,
);
const rect = landmarks.align();
const { imageDims } = sourceObj.detection;
const alignedRect = new FaceDetection(sourceObj.detection.score, rect.rescale(imageDims.reverse()), imageDims);
const alignedRect = new FaceDetection(
sourceObj.detection.score,
rect.rescale(imageDims.reverse()),
imageDims,
);
const angle = calculateFaceAngle(unshiftedLandmarks);
const extension = {