ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

finished draw buffering and smoothing and enabled by default

pull/134/head
Vladimir Mandic 2021-05-30 23:21:48 -04:00
parent 7b8bc7c687
commit 637f53736e
20 changed files with 12216 additions and 47464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
TODO.md
View File

@ -6,7 +6,6 @@ N/A
## Exploring Features
- Implement demo as installable PWA with model caching
- Implement results interpolation on library level instead inside draw functions
- Switch to TypeScript 4.3
- Unify score/confidence variables
@ -17,8 +16,6 @@ N/A
## In Progress
- Object detection interpolation
## Issues
- CenterNet with WebGL: <https://github.com/tensorflow/tfjs/issues/5145>

9
demo/index.js
View File

@ -29,7 +29,7 @@ let human;
const userConfig = {
warmup: 'none',
backend: 'webgl',
backend: 'humangl',
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/',
/*
async: false,
@ -38,7 +38,7 @@ const userConfig = {
enabled: false,
flip: false,
},
face: { enabled: true,
face: { enabled: false,
detector: { return: true },
mesh: { enabled: true },
iris: { enabled: true },
@ -49,15 +49,14 @@ const userConfig = {
// body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' },
body: { enabled: false, modelPath: 'movenet-lightning.json' },
object: { enabled: false },
object: { enabled: true },
gesture: { enabled: true },
*/
};
const drawOptions = {
bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement
bufferedOutput: true, // makes draw functions interpolate results between each detection for smoother movement
bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
drawGaze: true,
};
// ui options

4
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

40
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

40
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

14864
dist/human.node-gpu.js vendored
View File

File diff suppressed because it is too large Load Diff

14864
dist/human.node-wasm.js vendored
View File

File diff suppressed because it is too large Load Diff

14864
dist/human.node.js vendored
View File

File diff suppressed because it is too large Load Diff

34
server/build.log
View File

@ -1,17 +1,17 @@
2021-05-30 18:44:01 INFO:  @vladmandic/human version 2.0.0
2021-05-30 18:44:01 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-30 18:44:01 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-30 18:44:01 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: node type: node: {"imports":39,"importBytes":446470,"outputBytes":398046,"outputFiles":"dist/human.node.js"}
2021-05-30 18:44:01 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":446478,"outputBytes":398050,"outputFiles":"dist/human.node-gpu.js"}
2021-05-30 18:44:01 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":446545,"outputBytes":398122,"outputFiles":"dist/human.node-wasm.js"}
2021-05-30 18:44:01 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:02 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":446572,"outputBytes":243298,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-30 18:44:02 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:02 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1556596,"outputBytes":1351115,"outputFiles":"dist/human.js"}
2021-05-30 18:44:03 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1556596,"outputBytes":1351107,"outputFiles":"dist/human.esm.js"}
2021-05-30 18:44:03 INFO:  Generate types: ["src/human.ts"]
2021-05-30 18:44:07 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-30 18:44:07 INFO:  Generate TypeDocs: ["src/human.ts"]
2021-05-30 23:20:36 INFO:  @vladmandic/human version 2.0.0
2021-05-30 23:20:36 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-30 23:20:36 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-30 23:20:36 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 23:20:36 STATE: Build for: node type: node: {"imports":39,"importBytes":415373,"outputBytes":369593,"outputFiles":"dist/human.node.js"}
2021-05-30 23:20:36 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 23:20:36 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":415381,"outputBytes":369597,"outputFiles":"dist/human.node-gpu.js"}
2021-05-30 23:20:36 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 23:20:36 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":415448,"outputBytes":369669,"outputFiles":"dist/human.node-wasm.js"}
2021-05-30 23:20:37 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 23:20:37 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":415475,"outputBytes":243743,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-30 23:20:37 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 23:20:37 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1525499,"outputBytes":1351568,"outputFiles":"dist/human.js"}
2021-05-30 23:20:38 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1525499,"outputBytes":1351560,"outputFiles":"dist/human.esm.js"}
2021-05-30 23:20:38 INFO:  Generate types: ["src/human.ts"]
2021-05-30 23:20:43 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-30 23:20:43 INFO:  Generate TypeDocs: ["src/human.ts"]

181
src/draw/draw.ts
View File

@ -61,12 +61,12 @@ export const options: DrawOptions = {
drawLabels: <boolean>true,
drawBoxes: <boolean>true,
drawPolygons: <boolean>true,
drawGaze: <boolean>false,
drawGaze: <boolean>true,
fillPolygons: <boolean>false,
useDepth: <boolean>true,
useCurves: <boolean>false,
bufferedFactor: <number>2,
bufferedOutput: <boolean>false,
bufferedFactor: <number>3,
bufferedOutput: <boolean>true,
};
let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
@ -183,14 +183,14 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`);
// if (f.genderConfidence) labels.push(f.gender);
if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`iris distance: ${f.iris}`);
if (f.iris) labels.push(`distance: ${f.iris}`);
if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' '));
}
if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`);
if (f.rotation.gaze.bearing) labels.push(`gaze: ${rad2deg(f.rotation.gaze.bearing)}°`);
}
if (labels.length === 0) labels.push('face');
ctx.fillStyle = localOptions.color;
@ -245,10 +245,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.fill();
}
}
if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) {
if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.bearing) {
const leftGaze = [
f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]),
f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]),
f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
];
ctx.beginPath();
ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]);
@ -257,8 +257,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.stroke();
const rightGaze = [
f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]),
f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]),
f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
];
ctx.beginPath();
ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]);
@ -507,83 +507,108 @@ export async function person(inCanvas: HTMLCanvasElement, result: Array<Person>,
}
}
function calcBuffered(newResult, localOptions) {
function calcBuffered(newResult: Result, localOptions) {
// each record is only updated using deep clone when number of detected record changes, otherwise it will converge by itself
// otherwise bufferedResult is a shallow clone of result plus updated local calculated values
// thus mixing by-reference and by-value assignments to minimize memory operations
// interpolate body results
if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) bufferedResult.body = JSON.parse(JSON.stringify(newResult.body)); // deep clone once
for (let i = 0; i < newResult.body.length; i++) {
const box = newResult.body[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({
score: keypoint.score,
part: keypoint.part,
position: {
x: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / localOptions.bufferedFactor : keypoint.position.x,
y: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / localOptions.bufferedFactor : keypoint.position.y,
},
}));
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) {
bufferedResult.body = JSON.parse(JSON.stringify(newResult.body)); // deep clone once
} else {
for (let i = 0; i < newResult.body.length; i++) {
const box = newResult.body[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints
.map((keypoint, j) => ({
score: keypoint.score,
part: keypoint.part,
position: {
x: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / localOptions.bufferedFactor : keypoint.position.x,
y: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / localOptions.bufferedFactor : keypoint.position.y,
},
}));
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
}
}
// interpolate hand results
if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand)); // deep clone once
for (let i = 0; i < newResult.hand.length; i++) {
const box = newResult.hand[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw = newResult.hand[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / localOptions.bufferedFactor);
const landmarks = newResult.hand[i].landmarks // update landmarks
.map((landmark, j) => landmark
.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].landmarks[j][k] + coord) / localOptions.bufferedFactor));
const keys = Object.keys(newResult.hand[i].annotations); // update annotations
const annotations = [];
for (const key of keys) {
annotations[key] = newResult.hand[i].annotations[key]
.map((val, j) => val
.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / localOptions.bufferedFactor));
if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) {
bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand)); // deep clone once
} else {
for (let i = 0; i < newResult.hand.length; i++) {
const box = (newResult.hand[i].box// update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const landmarks = newResult.hand[i].landmarks // update landmarks
.map((landmark, j) => landmark
.map((coord, k) => (((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].landmarks[j][k] + coord) / localOptions.bufferedFactor)) as [number, number, number]);
const keys = Object.keys(newResult.hand[i].annotations); // update annotations
const annotations = {};
for (const key of keys) {
annotations[key] = newResult.hand[i].annotations[key]
.map((val, j) => val.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / localOptions.bufferedFactor));
}
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, landmarks, annotations }; // shallow clone plus updated values
}
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, landmarks, annotations }; // shallow clone plus updated values
}
// interpolate face results
if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) bufferedResult.face = JSON.parse(JSON.stringify(newResult.face)); // deep clone once
for (let i = 0; i < newResult.face.length; i++) {
const box = newResult.face[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw = newResult.face[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor);
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
};
const gaze = {
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor,
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) {
bufferedResult.face = JSON.parse(JSON.stringify(newResult.face)); // deep clone once
} else {
for (let i = 0; i < newResult.face.length; i++) {
const box = (newResult.face[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const boxRaw = (newResult.face[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const matrix = newResult.face[i].rotation.matrix;
const angle = {
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
};
const gaze = {
bearing: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.bearing + newResult.face[i].rotation.gaze.bearing) / localOptions.bufferedFactor, // not correct due to wrap-around
/*
angle: Math.atan2( // average angle is calculated differently
Math.sin(bufferedResult.face[i].rotation.gaze.angle) + Math.sin(newResult.face[i].rotation.gaze.angle),
Math.cos(bufferedResult.face[i].rotation.gaze.angle) + Math.sin(newResult.face[i].rotation.gaze.angle),
),
*/
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
}
}
// interpolate object detection results
if (!bufferedResult.object || (newResult.object.length !== bufferedResult.object.length)) {
bufferedResult.object = JSON.parse(JSON.stringify(newResult.object)); // deep clone once
} else {
for (let i = 0; i < newResult.object.length; i++) {
const box = newResult.object[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw = newResult.object[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.object[i] = { ...newResult.object[i], box, boxRaw }; // shallow clone plus updated values
}
}
// interpolate person results
const newPersons = newResult.persons; // trigger getter function
if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) bufferedResult.persons = JSON.parse(JSON.stringify(newPersons));
for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow
bufferedResult.persons[i].box = newPersons[i].box
.map((box, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / localOptions.bufferedFactor);
if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) {
bufferedResult.persons = JSON.parse(JSON.stringify(newPersons));
} else {
for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow
bufferedResult.persons[i].box = (newPersons[i].box
.map((box, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / localOptions.bufferedFactor)) as [number, number, number, number];
}
}
// no buffering implemented for face, object, gesture
// bufferedResult.face = JSON.parse(JSON.stringify(newResult.face));
// bufferedResult.object = JSON.parse(JSON.stringify(newResult.object));
// bufferedResult.gesture = JSON.parse(JSON.stringify(newResult.gesture));
}
export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasElement) {
@ -597,12 +622,14 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio
const localOptions = mergeDeep(options, drawOptions);
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
if (localOptions.bufferedOutput) calcBuffered(result, localOptions); // do results interpolation
else bufferedResult = result; // just use results as-is
face(inCanvas, bufferedResult.face, localOptions); // face does have buffering
body(inCanvas, bufferedResult.body, localOptions); // use interpolated results if available
hand(inCanvas, bufferedResult.hand, localOptions); // use interpolated results if available
// person(inCanvas, bufferedResult.persons, localOptions); // use interpolated results if available
if (!bufferedResult) bufferedResult = result; // first pass
else if (localOptions.bufferedOutput) calcBuffered(result, localOptions); // do results interpolation
else bufferedResult = result; // or just use results as-is
face(inCanvas, bufferedResult.face, localOptions);
body(inCanvas, bufferedResult.body, localOptions);
hand(inCanvas, bufferedResult.hand, localOptions);
object(inCanvas, bufferedResult.object, localOptions);
// person(inCanvas, bufferedResult.persons, localOptions);
gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering
object(inCanvas, result.object, localOptions); // object detection does not have buffering
}

14
src/face.ts
View File

@ -12,7 +12,7 @@ import { Face } from './result';
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const rad2deg = (theta) => (theta * 180) / Math.PI;
const calculateGaze = (mesh): { angle: number, strength: number } => {
const calculateGaze = (mesh, box): { bearing: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
@ -31,17 +31,17 @@ const calculateGaze = (mesh): { angle: number, strength: number } => {
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1],
];
const vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
let vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
vectorLength = Math.min(vectorLength, box[2] / 2, box[3] / 2); // limit strength to half of box size
const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments
// vectorAngle right=0*pi, up=1*pi/2, left=1*pi, down=3*pi/2
return { angle: vectorAngle, strength: vectorLength };
return { bearing: vectorAngle, strength: vectorLength };
};
const calculateFaceAngle = (face, imageSize): {
angle: { pitch: number, yaw: number, roll: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number },
gaze: { bearing: number, strength: number },
} => {
// const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360);
const normalize = (v) => { // normalize vector
@ -104,7 +104,7 @@ const calculateFaceAngle = (face, imageSize): {
// initialize gaze and mesh
const mesh = face.meshRaw;
if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { angle: 0, strength: 0 } };
if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { bearing: 0, strength: 0 } };
const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5;
// top, bottom, left, right
@ -132,7 +132,7 @@ const calculateFaceAngle = (face, imageSize): {
// const angle = meshToEulerAngle(mesh);
// we have iris keypoints so we can calculate gaze direction
const gaze = mesh.length === 478 ? calculateGaze(mesh) : { angle: 0, strength: 0 };
const gaze = mesh.length === 478 ? calculateGaze(mesh, face.box) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze };
};

14720
src/handpose/anchors.ts
View File

File diff suppressed because it is too large Load Diff

2
src/handpose/handpose.ts
View File

@ -35,7 +35,7 @@ export async function predict(input, config): Promise<Hand[]> {
}
}
const landmarks = predictions[i].landmarks as number[];
const landmarks = predictions[i].landmarks as unknown as Array<[number, number, number]>;
let box: [number, number, number, number] = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work
let boxRaw: [number, number, number, number] = [0, 0, 0, 0];

8
src/result.ts
View File

@ -28,7 +28,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations
* - angle: face angle as object with values for roll, yaw and pitch angles
* - matrix: 3d transofrmation matrix as array of numeric values
* - gaze: gaze direction as object with values for agngle in radians and strengthss
* - gaze: gaze direction as object with values for bearing in radians and relative strength
* - tensor: face tensor as Tensor object which contains detected face
*/
export interface Face {
@ -50,7 +50,7 @@ export interface Face {
rotation: {
angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number },
gaze: { bearing: number, strength: number },
}
tensor: typeof Tensor,
}
@ -97,8 +97,8 @@ export interface Hand {
confidence: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
landmarks: number[],
annotations: Record<string, Array<{ part: string, points: Array<[number, number, number]> }>>,
landmarks: Array<[number, number, number]>,
annotations: Record<string, Array<[number, number, number]>>,
}
/** Object results

2
typedoc/assets/js/search.js
View File

File diff suppressed because one or more lines are too long

8
typedoc/interfaces/face.html
View File

@ -91,7 +91,7 @@
<li>rotation: face rotiation that contains both angles and matrix used for 3d transformations</li>
<li>angle: face angle as object with values for roll, yaw and pitch angles</li>
<li>matrix: 3d transofrmation matrix as array of numeric values</li>
<li>gaze: gaze direction as object with values for agngle in radians and strengthss</li>
<li>gaze: gaze direction as object with values for bearing in radians and relative strength</li>
<li>tensor: face tensor as Tensor object which contains detected face</li>
</ul>
</div>
@ -243,7 +243,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="rotation" class="tsd-anchor"></a>
<h3>rotation</h3>
<div class="tsd-signature tsd-kind-icon">rotation<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>pitch<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>roll<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>yaw<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>matrix<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol"> }</span></div>
<div class="tsd-signature tsd-kind-icon">rotation<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>pitch<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>roll<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>yaw<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>matrix<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources">
</aside>
<div class="tsd-type-declaration">
@ -264,10 +264,10 @@
</ul>
</li>
<li class="tsd-parameter">
<h5>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></h5>
<h5>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></h5>
<ul class="tsd-parameters">
<li class="tsd-parameter">
<h5>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
<h5>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li>
<li class="tsd-parameter">
<h5>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>

4
typedoc/interfaces/hand.html
View File

@ -110,7 +110,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="annotations" class="tsd-anchor"></a>
<h3>annotations</h3>
<div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol">&lt;</span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">{ </span>part<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>points<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">&gt;</span></div>
<div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol">&lt;</span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">&gt;</span></div>
<aside class="tsd-sources">
</aside>
</section>
@ -145,7 +145,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="landmarks" class="tsd-anchor"></a>
<h3>landmarks</h3>
<div class="tsd-signature tsd-kind-icon">landmarks<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">[]</span></div>
<div class="tsd-signature tsd-kind-icon">landmarks<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span></div>
<aside class="tsd-sources">
</aside>
</section>

11
types/result.d.ts vendored
View File

@ -26,7 +26,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations
* - angle: face angle as object with values for roll, yaw and pitch angles
* - matrix: 3d transofrmation matrix as array of numeric values
* - gaze: gaze direction as object with values for agngle in radians and strengthss
* - gaze: gaze direction as object with values for bearing in radians and relative strength
* - tensor: face tensor as Tensor object which contains detected face
*/
export interface Face {
@ -59,7 +59,7 @@ export interface Face {
};
matrix: [number, number, number, number, number, number, number, number, number];
gaze: {
angle: number;
bearing: number;
strength: number;
};
};
@ -114,11 +114,8 @@ export interface Hand {
confidence: number;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
landmarks: number[];
annotations: Record<string, Array<{
part: string;
points: Array<[number, number, number]>;
}>>;
landmarks: Array<[number, number, number]>;
annotations: Record<string, Array<[number, number, number]>>;
}
/** Object results
*