diff --git a/demo/webcam.js b/demo/webcam.js index f2dad27..039346a 100644 --- a/demo/webcam.js +++ b/demo/webcam.js @@ -53,12 +53,12 @@ function drawFaces(canvas, data, fps) { ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 59); ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 41); ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 23); - ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 5); + ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 5); ctx.fillStyle = 'lightblue'; ctx.fillText(`gender: ${Math.round(100 * person.genderProbability)}% ${person.gender}`, person.detection.box.x, person.detection.box.y - 60); ctx.fillText(`expression: ${Math.round(100 * expression[0][1])}% ${expression[0][0]}`, person.detection.box.x, person.detection.box.y - 42); ctx.fillText(`age: ${Math.round(person.age)} years`, person.detection.box.x, person.detection.box.y - 24); - ctx.fillText(`roll:${person.angle.roll.toFixed(3)} pitch:${person.angle.pitch.toFixed(3)} yaw:${person.angle.yaw.toFixed(3)}`, person.detection.box.x, person.detection.box.y - 6); + ctx.fillText(`roll:${person.angle.roll}° pitch:${person.angle.pitch}° yaw:${person.angle.yaw}°`, person.detection.box.x, person.detection.box.y - 6); // draw face points for each face ctx.globalAlpha = 0.8; ctx.fillStyle = 'lightblue'; diff --git a/src/factories/WithFaceLandmarks.ts b/src/factories/WithFaceLandmarks.ts index b56413c..a695ec1 100644 --- a/src/factories/WithFaceLandmarks.ts +++ b/src/factories/WithFaceLandmarks.ts @@ -5,64 +5,130 @@ import { isWithFaceDetection, WithFaceDetection } from './WithFaceDetection'; export type WithFaceLandmarks< TSource extends WithFaceDetection<{}>, - TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 > = TSource & { - landmarks: TFaceLandmarks, - unshiftedLandmarks: TFaceLandmarks, - alignedRect: FaceDetection, - angle: { roll: number | undefined, pitch: number | undefined, yaw: number | undefined }, - } + TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 +> = TSource & { + landmarks: TFaceLandmarks; + unshiftedLandmarks: TFaceLandmarks; + alignedRect: FaceDetection; + angle: { + roll: number | undefined; + pitch: number | undefined; + yaw: number | undefined; + }; +}; -export function isWithFaceLandmarks(obj: any): obj is WithFaceLandmarks, FaceLandmarks> { - return isWithFaceDetection(obj) +export function isWithFaceLandmarks( + obj: any, +): obj is WithFaceLandmarks, FaceLandmarks> { + return ( + isWithFaceDetection(obj) // eslint-disable-next-line dot-notation && obj['landmarks'] instanceof FaceLandmarks // eslint-disable-next-line dot-notation && obj['unshiftedLandmarks'] instanceof FaceLandmarks // eslint-disable-next-line dot-notation - && obj['alignedRect'] instanceof FaceDetection; + && obj['alignedRect'] instanceof FaceDetection + ); } function calculateFaceAngle(mesh) { - // returns the angle in the plane (in radians) between the positive x-axis and the ray from (0,0) to the point (x,y) - const radians = (a1, a2, b1, b2) => (Math.atan2(b2 - a2, b1 - a1) % Math.PI); - // convert radians to degrees - // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars - const degrees = (theta) => (theta * 180) / Math.PI; + /* + AUTHORED BY: SOHAIB AHMED + https://github.com/TheSohaibAhmed/ + */ - const angle = { roll: undefined, pitch: undefined, yaw: undefined }; + // Helper to convert radians to degrees + // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars + const degrees = (radians) => (radians * 180) / Math.PI; + + const angle = { + roll: undefined, + pitch: undefined, + yaw: undefined, + }; if (!mesh || !mesh._positions || mesh._positions.length !== 68) return angle; const pt = mesh._positions; - // values are in radians in range of -pi/2 to pi/2 which is -90 to +90 degrees - // value of 0 means center + function calcLengthBetweenTwoPoints(a, b) { + return Math.sqrt((a._x - b._x) ** 2 + (a._y - b._y) ** 2); + } + const calcYaw = (leftPoint, midPoint, rightPoint) => { + // Calc x-distance from left side of the face ("ear") to facial midpoint ("nose") + const leftToMidpoint = Math.floor(leftPoint._x - midPoint._x); - // roll is face lean from left to right - // comparing x,y of outside corners of leftEye and rightEye - angle.roll = -radians(pt[36]._x, pt[36]._y, pt[45]._x, pt[45]._y); + // Calc x-distance from facial midpoint ("nose") to the right side of the face ("ear") + const rightToMidpoint = Math.floor(midPoint._x - rightPoint._x); - // pitch is face turn from left right - // comparing x distance of top of nose to left and right edge of face - // precision is lacking since coordinates are not precise enough - angle.pitch = radians(0, Math.abs(pt[0]._x - pt[30]._x) / pt[30]._x, Math.PI, Math.abs(pt[16]._x - pt[30]._x) / pt[30]._x); + // Difference in distances coincidentally approximates to angles + const distanceApproximatesToAngle = leftToMidpoint - rightToMidpoint; + return distanceApproximatesToAngle; + }; - // yaw is face move from up to down - // comparing size of the box around the face with top and bottom of detected landmarks - // silly hack, but this gives us face compression on y-axis - // e.g., tilting head up hides the forehead that doesn't have any landmarks so ratio drops - const bottom = pt.reduce((prev, cur) => (prev < cur._y ? prev : cur._y), +Infinity); - const top = pt.reduce((prev, cur) => (prev > cur._y ? prev : cur._y), -Infinity); - angle.yaw = Math.PI * (mesh._imgDims._height / (top - bottom) / 1.40 - 1); + const calcRoll = (lever, pivot) => { + // When rolling, the head seems to pivot from the nose/lips/chin area. + // So, we'll choose any two points from the facial midline, where the first point should be the pivot, and the other "lever" + // Plan/Execution: get the hypotenuse & opposite sides of a 90deg triangle ==> Calculate angle in radians + + const hypotenuse = Math.hypot(pivot._x - lever._x, pivot._y - lever._y); + const opposite = pivot._y - lever._y; + const angleInRadians = Math.asin(opposite / hypotenuse); + const angleInDegrees = degrees(angleInRadians); + const normalizeAngle = Math.floor(90 - angleInDegrees); + // If lever more to the left of the pivot, then we're tilting left + // "-" is negative direction. "+", or absence of a sign is positive direction + const tiltDirection = pivot._x - lever._x < 0 ? -1 : 1; + const result = normalizeAngle * tiltDirection; + return result; + }; + + const calcPitch = (leftPoint, midPoint, rightPoint) => { + // Theory: While pitching, the nose is the most salient point --> That's what we'll use to make a trianle. + // The "base" is between point that don't move when we pitch our head (i.e. an imaginary line running ear to ear through the nose). + // Executuin: Get the opposite & adjacent lengths of the triangle from the ear's perspective. Use it to get angle. + + const base = calcLengthBetweenTwoPoints(leftPoint, rightPoint); + // adjecent is base/2 technically. + const baseCoords = { + _x: (leftPoint._x + rightPoint._x) / 2, + _y: (leftPoint._y + rightPoint._y) / 2, + }; + const midToBaseLength = calcLengthBetweenTwoPoints(midPoint, baseCoords); + const angleInRadians = Math.atan(midToBaseLength / base); + const angleInDegrees = Math.floor(degrees(angleInRadians)); + // Account for directionality. + // pitch forwards (_i.e. tilting your head forwards) is positive (or no sign); backward is negative. + const direction = baseCoords._y - midPoint._y < 0 ? -1 : 1; + const result = angleInDegrees * direction; + return result; + }; + + angle.roll = calcRoll(pt[27], pt[66]); + angle.pitch = calcPitch(pt[14], pt[30], pt[2]); + angle.yaw = calcYaw(pt[14], pt[33], pt[2]); return angle; } -export function extendWithFaceLandmarks, TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 >(sourceObj: TSource, unshiftedLandmarks: TFaceLandmarks): WithFaceLandmarks { +export function extendWithFaceLandmarks< + TSource extends WithFaceDetection<{}>, + TFaceLandmarks extends FaceLandmarks = FaceLandmarks68 +>( + sourceObj: TSource, + unshiftedLandmarks: TFaceLandmarks, +): WithFaceLandmarks { const { box: shift } = sourceObj.detection; - const landmarks = unshiftedLandmarks.shiftBy(shift.x, shift.y); + const landmarks = unshiftedLandmarks.shiftBy( + shift.x, + shift.y, + ); const rect = landmarks.align(); const { imageDims } = sourceObj.detection; - const alignedRect = new FaceDetection(sourceObj.detection.score, rect.rescale(imageDims.reverse()), imageDims); + const alignedRect = new FaceDetection( + sourceObj.detection.score, + rect.rescale(imageDims.reverse()), + imageDims, + ); const angle = calculateFaceAngle(unshiftedLandmarks); const extension = {