ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update gaze strength calculations

pull/134/head
Vladimir Mandic 2021-06-03 09:53:11 -04:00
parent 8a95539741
commit 748540a0cc
12 changed files with 200892 additions and 4515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10916
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

67535
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

67547
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

22
dist/human.node-gpu.js vendored
View File

@ -174,7 +174,7 @@ var config = {
description: { description: {
enabled: true, enabled: true,
modelPath: "faceres.json", modelPath: "faceres.json",
skipFrames: 16, skipFrames: 11,
minConfidence: 0.1 minConfidence: 0.1
}, },
emotion: { emotion: {
@ -189,12 +189,12 @@ var config = {
modelPath: "movenet-lightning.json", modelPath: "movenet-lightning.json",
maxDetected: 1, maxDetected: 1,
minConfidence: 0.2, minConfidence: 0.2,
skipFrames: 16 skipFrames: 1
}, },
hand: { hand: {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 19, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -212,7 +212,7 @@ var config = {
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
skipFrames: 20 skipFrames: 19
} }
}; };
@ -4332,20 +4332,20 @@ async function predict3(image15, config3, idx, count2) {
} }
// src/face.ts // src/face.ts
var calculateGaze = (mesh, box6) => { var calculateGaze = (face5) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, -0.1]; const offsetIris = [0, -0.1];
const eyeRatio = 1; const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2]; const left = face5.mesh[33][2] > face5.mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468]; const irisCenter = left ? face5.mesh[473] : face5.mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; const eyeCenter = left ? [(face5.mesh[133][0] + face5.mesh[33][0]) / 2, (face5.mesh[133][1] + face5.mesh[33][1]) / 2] : [(face5.mesh[263][0] + face5.mesh[362][0]) / 2, (face5.mesh[263][1] + face5.mesh[362][1]) / 2];
const eyeSize = left ? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] : [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; const eyeSize = left ? [face5.mesh[133][0] - face5.mesh[33][0], face5.mesh[23][1] - face5.mesh[27][1]] : [face5.mesh[263][0] - face5.mesh[362][0], face5.mesh[253][1] - face5.mesh[257][1]];
const eyeDiff = [ const eyeDiff = [
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1] eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1]
]; ];
let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2); let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2);
strength = Math.min(strength, box6[2] / 2, box6[3] / 2); strength = Math.min(strength, face5.boxRaw[2] / 2, face5.boxRaw[3] / 2);
const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI; const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI;
return { bearing, strength }; return { bearing, strength };
}; };
@ -4425,7 +4425,7 @@ var calculateFaceAngle = (face5, imageSize) => {
z_axis[2] z_axis[2]
]; ];
const angle = rotationMatrixToEulerAngle(matrix); const angle = rotationMatrixToEulerAngle(matrix);
const gaze = mesh.length === 478 ? calculateGaze(mesh, face5.box) : { bearing: 0, strength: 0 }; const gaze = mesh.length === 478 ? calculateGaze(face5) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze }; return { angle, matrix, gaze };
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {

22
dist/human.node-wasm.js vendored
View File

@ -175,7 +175,7 @@ var config = {
description: { description: {
enabled: true, enabled: true,
modelPath: "faceres.json", modelPath: "faceres.json",
skipFrames: 16, skipFrames: 11,
minConfidence: 0.1 minConfidence: 0.1
}, },
emotion: { emotion: {
@ -190,12 +190,12 @@ var config = {
modelPath: "movenet-lightning.json", modelPath: "movenet-lightning.json",
maxDetected: 1, maxDetected: 1,
minConfidence: 0.2, minConfidence: 0.2,
skipFrames: 16 skipFrames: 1
}, },
hand: { hand: {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 19, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -213,7 +213,7 @@ var config = {
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
skipFrames: 20 skipFrames: 19
} }
}; };
@ -4333,20 +4333,20 @@ async function predict3(image15, config3, idx, count2) {
} }
// src/face.ts // src/face.ts
var calculateGaze = (mesh, box6) => { var calculateGaze = (face5) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, -0.1]; const offsetIris = [0, -0.1];
const eyeRatio = 1; const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2]; const left = face5.mesh[33][2] > face5.mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468]; const irisCenter = left ? face5.mesh[473] : face5.mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; const eyeCenter = left ? [(face5.mesh[133][0] + face5.mesh[33][0]) / 2, (face5.mesh[133][1] + face5.mesh[33][1]) / 2] : [(face5.mesh[263][0] + face5.mesh[362][0]) / 2, (face5.mesh[263][1] + face5.mesh[362][1]) / 2];
const eyeSize = left ? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] : [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; const eyeSize = left ? [face5.mesh[133][0] - face5.mesh[33][0], face5.mesh[23][1] - face5.mesh[27][1]] : [face5.mesh[263][0] - face5.mesh[362][0], face5.mesh[253][1] - face5.mesh[257][1]];
const eyeDiff = [ const eyeDiff = [
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1] eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1]
]; ];
let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2); let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2);
strength = Math.min(strength, box6[2] / 2, box6[3] / 2); strength = Math.min(strength, face5.boxRaw[2] / 2, face5.boxRaw[3] / 2);
const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI; const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI;
return { bearing, strength }; return { bearing, strength };
}; };
@ -4426,7 +4426,7 @@ var calculateFaceAngle = (face5, imageSize) => {
z_axis[2] z_axis[2]
]; ];
const angle = rotationMatrixToEulerAngle(matrix); const angle = rotationMatrixToEulerAngle(matrix);
const gaze = mesh.length === 478 ? calculateGaze(mesh, face5.box) : { bearing: 0, strength: 0 }; const gaze = mesh.length === 478 ? calculateGaze(face5) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze }; return { angle, matrix, gaze };
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {

22
dist/human.node.js vendored
View File

@ -174,7 +174,7 @@ var config = {
description: { description: {
enabled: true, enabled: true,
modelPath: "faceres.json", modelPath: "faceres.json",
skipFrames: 16, skipFrames: 11,
minConfidence: 0.1 minConfidence: 0.1
}, },
emotion: { emotion: {
@ -189,12 +189,12 @@ var config = {
modelPath: "movenet-lightning.json", modelPath: "movenet-lightning.json",
maxDetected: 1, maxDetected: 1,
minConfidence: 0.2, minConfidence: 0.2,
skipFrames: 16 skipFrames: 1
}, },
hand: { hand: {
enabled: true, enabled: true,
rotation: true, rotation: true,
skipFrames: 19, skipFrames: 18,
minConfidence: 0.1, minConfidence: 0.1,
iouThreshold: 0.1, iouThreshold: 0.1,
maxDetected: 2, maxDetected: 2,
@ -212,7 +212,7 @@ var config = {
minConfidence: 0.2, minConfidence: 0.2,
iouThreshold: 0.4, iouThreshold: 0.4,
maxDetected: 10, maxDetected: 10,
skipFrames: 20 skipFrames: 19
} }
}; };
@ -4332,20 +4332,20 @@ async function predict3(image15, config3, idx, count2) {
} }
// src/face.ts // src/face.ts
var calculateGaze = (mesh, box6) => { var calculateGaze = (face5) => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]);
const offsetIris = [0, -0.1]; const offsetIris = [0, -0.1];
const eyeRatio = 1; const eyeRatio = 1;
const left = mesh[33][2] > mesh[263][2]; const left = face5.mesh[33][2] > face5.mesh[263][2];
const irisCenter = left ? mesh[473] : mesh[468]; const irisCenter = left ? face5.mesh[473] : face5.mesh[468];
const eyeCenter = left ? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] : [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; const eyeCenter = left ? [(face5.mesh[133][0] + face5.mesh[33][0]) / 2, (face5.mesh[133][1] + face5.mesh[33][1]) / 2] : [(face5.mesh[263][0] + face5.mesh[362][0]) / 2, (face5.mesh[263][1] + face5.mesh[362][1]) / 2];
const eyeSize = left ? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] : [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; const eyeSize = left ? [face5.mesh[133][0] - face5.mesh[33][0], face5.mesh[23][1] - face5.mesh[27][1]] : [face5.mesh[263][0] - face5.mesh[362][0], face5.mesh[253][1] - face5.mesh[257][1]];
const eyeDiff = [ const eyeDiff = [
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1] eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1]
]; ];
let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2); let strength = Math.sqrt(eyeDiff[0] ** 2 + eyeDiff[1] ** 2);
strength = Math.min(strength, box6[2] / 2, box6[3] / 2); strength = Math.min(strength, face5.boxRaw[2] / 2, face5.boxRaw[3] / 2);
const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI; const bearing = (radians([0, 0], eyeDiff) + Math.PI / 2) % Math.PI;
return { bearing, strength }; return { bearing, strength };
}; };
@ -4425,7 +4425,7 @@ var calculateFaceAngle = (face5, imageSize) => {
z_axis[2] z_axis[2]
]; ];
const angle = rotationMatrixToEulerAngle(matrix); const angle = rotationMatrixToEulerAngle(matrix);
const gaze = mesh.length === 478 ? calculateGaze(mesh, face5.box) : { bearing: 0, strength: 0 }; const gaze = mesh.length === 478 ? calculateGaze(face5) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze }; return { angle, matrix, gaze };
}; };
var detectFace = async (parent, input) => { var detectFace = async (parent, input) => {

59233
dist/tfjs.esm.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/tfjs.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
src/config.ts
View File

@ -280,7 +280,7 @@ const config: Config = {
// recommended to enable detector.rotation and mesh.enabled // recommended to enable detector.rotation and mesh.enabled
modelPath: 'faceres.json', // face description model modelPath: 'faceres.json', // face description model
// can be either absolute path or relative to modelBasePath // can be either absolute path or relative to modelBasePath
skipFrames: 16, // how many max frames to go without re-running the detector skipFrames: 11, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
}, },
@ -302,7 +302,7 @@ const config: Config = {
// should be set to the minimum number for performance // should be set to the minimum number for performance
// only valid for posenet as other models detects single pose // only valid for posenet as other models detects single pose
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
skipFrames: 16, // how many max frames to go without re-running the detector skipFrames: 1, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
}, },
@ -310,7 +310,7 @@ const config: Config = {
enabled: true, enabled: true,
rotation: true, // use best-guess rotated hand image or just box with rotation as-is rotation: true, // use best-guess rotated hand image or just box with rotation as-is
// false means higher performance, but incorrect finger mapping if hand is inverted // false means higher performance, but incorrect finger mapping if hand is inverted
skipFrames: 19, // how many max frames to go without re-running the hand bounding box detector skipFrames: 18, // how many max frames to go without re-running the hand bounding box detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
// e.g., if model is running st 25 FPS, we can re-use existing bounding // e.g., if model is running st 25 FPS, we can re-use existing bounding
// box for updated hand skeleton analysis as the hand probably // box for updated hand skeleton analysis as the hand probably
@ -335,7 +335,7 @@ const config: Config = {
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.4, // ammount of overlap between two detected objects before one object is removed
maxDetected: 10, // maximum number of objects detected in the input maxDetected: 10, // maximum number of objects detected in the input
skipFrames: 20, // how many max frames to go without re-running the detector skipFrames: 19, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
}, },
}; };

18
src/face.ts
View File

@ -14,27 +14,27 @@ import { Tensor } from './tfjs/types';
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const rad2deg = (theta) => Math.round((theta * 180) / Math.PI); const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
const calculateGaze = (mesh, box): { bearing: number, strength: number } => { const calculateGaze = (face): { bearing: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
const eyeRatio = 1; // factor to normalize changes x vs y const eyeRatio = 1; // factor to normalize changes x vs y
const left = mesh[33][2] > mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis const left = face.mesh[33][2] > face.mesh[263][2]; // pick left or right eye depending which one is closer bazed on outsize point z axis
const irisCenter = left ? mesh[473] : mesh[468]; const irisCenter = left ? face.mesh[473] : face.mesh[468];
const eyeCenter = left // eye center is average of extreme points on x axis for both x and y, ignoring y extreme points as eyelids naturally open/close more when gazing up/down so relative point is less precise const eyeCenter = left // eye center is average of extreme points on x axis for both x and y, ignoring y extreme points as eyelids naturally open/close more when gazing up/down so relative point is less precise
? [(mesh[133][0] + mesh[33][0]) / 2, (mesh[133][1] + mesh[33][1]) / 2] ? [(face.mesh[133][0] + face.mesh[33][0]) / 2, (face.mesh[133][1] + face.mesh[33][1]) / 2]
: [(mesh[263][0] + mesh[362][0]) / 2, (mesh[263][1] + mesh[362][1]) / 2]; : [(face.mesh[263][0] + face.mesh[362][0]) / 2, (face.mesh[263][1] + face.mesh[362][1]) / 2];
const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions const eyeSize = left // eye size is difference between extreme points for both x and y, used to normalize & squarify eye dimensions
? [mesh[133][0] - mesh[33][0], mesh[23][1] - mesh[27][1]] ? [face.mesh[133][0] - face.mesh[33][0], face.mesh[23][1] - face.mesh[27][1]]
: [mesh[263][0] - mesh[362][0], mesh[253][1] - mesh[257][1]]; : [face.mesh[263][0] - face.mesh[362][0], face.mesh[253][1] - face.mesh[257][1]];
const eyeDiff = [ // x distance between extreme point and center point normalized with eye size const eyeDiff = [ // x distance between extreme point and center point normalized with eye size
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1], eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1],
]; ];
let strength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences let strength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
strength = Math.min(strength, box[2] / 2, box[3] / 2); // limit strength to half of box size to avoid clipping due to low precision strength = Math.min(strength, face.boxRaw[2] / 2, face.boxRaw[3] / 2); // limit strength to half of box size to avoid clipping due to low precision
const bearing = (radians([0, 0], eyeDiff) + (Math.PI / 2)) % Math.PI; // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments and rotate clockwise 90degrees const bearing = (radians([0, 0], eyeDiff) + (Math.PI / 2)) % Math.PI; // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments and rotate clockwise 90degrees
return { bearing, strength }; return { bearing, strength };
@ -134,7 +134,7 @@ const calculateFaceAngle = (face, imageSize): {
// const angle = meshToEulerAngle(mesh); // const angle = meshToEulerAngle(mesh);
// we have iris keypoints so we can calculate gaze direction // we have iris keypoints so we can calculate gaze direction
const gaze = mesh.length === 478 ? calculateGaze(mesh, face.box) : { bearing: 0, strength: 0 }; const gaze = mesh.length === 478 ? calculateGaze(face) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze }; return { angle, matrix, gaze };
}; };