ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

finished draw buffering and smoothing and enabled by default

pull/134/head
Vladimir Mandic 2021-05-30 23:21:48 -04:00
parent 7b8bc7c687
commit 637f53736e
20 changed files with 12216 additions and 47464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
TODO.md
View File

@ -6,7 +6,6 @@ N/A
## Exploring Features ## Exploring Features
- Implement demo as installable PWA with model caching
- Implement results interpolation on library level instead inside draw functions - Implement results interpolation on library level instead inside draw functions
- Switch to TypeScript 4.3 - Switch to TypeScript 4.3
- Unify score/confidence variables - Unify score/confidence variables
@ -17,8 +16,6 @@ N/A
## In Progress ## In Progress
- Object detection interpolation
## Issues ## Issues
- CenterNet with WebGL: <https://github.com/tensorflow/tfjs/issues/5145> - CenterNet with WebGL: <https://github.com/tensorflow/tfjs/issues/5145>

9
demo/index.js
View File

@ -29,7 +29,7 @@ let human;
const userConfig = { const userConfig = {
warmup: 'none', warmup: 'none',
backend: 'webgl', backend: 'humangl',
wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/', wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.6.0/dist/',
/* /*
async: false, async: false,
@ -38,7 +38,7 @@ const userConfig = {
enabled: false, enabled: false,
flip: false, flip: false,
}, },
face: { enabled: true, face: { enabled: false,
detector: { return: true }, detector: { return: true },
mesh: { enabled: true }, mesh: { enabled: true },
iris: { enabled: true }, iris: { enabled: true },
@ -49,15 +49,14 @@ const userConfig = {
// body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'posenet.json' },
// body: { enabled: true, modelPath: 'blazepose.json' }, // body: { enabled: true, modelPath: 'blazepose.json' },
body: { enabled: false, modelPath: 'movenet-lightning.json' }, body: { enabled: false, modelPath: 'movenet-lightning.json' },
object: { enabled: false }, object: { enabled: true },
gesture: { enabled: true }, gesture: { enabled: true },
*/ */
}; };
const drawOptions = { const drawOptions = {
bufferedOutput: true, // experimental feature that makes draw functions interpolate results between each detection for smoother movement bufferedOutput: true, // makes draw functions interpolate results between each detection for smoother movement
bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc. bufferedFactor: 4, // speed of interpolation convergence where 1 means 100% immediately, 2 means 50% at each interpolation, etc.
drawGaze: true,
}; };
// ui options // ui options

4
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

40
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

40
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

14864
dist/human.node-gpu.js vendored
View File

File diff suppressed because it is too large Load Diff

14864
dist/human.node-wasm.js vendored
View File

File diff suppressed because it is too large Load Diff

14864
dist/human.node.js vendored
View File

File diff suppressed because it is too large Load Diff

34
server/build.log
View File

@ -1,17 +1,17 @@
2021-05-30 18:44:01 INFO:  @vladmandic/human version 2.0.0 2021-05-30 23:20:36 INFO:  @vladmandic/human version 2.0.0
2021-05-30 18:44:01 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 2021-05-30 23:20:36 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0
2021-05-30 18:44:01 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} 2021-05-30 23:20:36 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-05-30 18:44:01 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"} 2021-05-30 23:20:36 STATE: Build for: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1292,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: node type: node: {"imports":39,"importBytes":446470,"outputBytes":398046,"outputFiles":"dist/human.node.js"} 2021-05-30 23:20:36 STATE: Build for: node type: node: {"imports":39,"importBytes":415373,"outputBytes":369593,"outputFiles":"dist/human.node.js"}
2021-05-30 18:44:01 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"} 2021-05-30 23:20:36 STATE: Build for: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1300,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":446478,"outputBytes":398050,"outputFiles":"dist/human.node-gpu.js"} 2021-05-30 23:20:36 STATE: Build for: nodeGPU type: node: {"imports":39,"importBytes":415381,"outputBytes":369597,"outputFiles":"dist/human.node-gpu.js"}
2021-05-30 18:44:01 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"} 2021-05-30 23:20:36 STATE: Build for: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1367,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:01 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":446545,"outputBytes":398122,"outputFiles":"dist/human.node-wasm.js"} 2021-05-30 23:20:36 STATE: Build for: nodeWASM type: node: {"imports":39,"importBytes":415448,"outputBytes":369669,"outputFiles":"dist/human.node-wasm.js"}
2021-05-30 18:44:01 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"} 2021-05-30 23:20:37 STATE: Build for: browserNoBundle type: tfjs: {"imports":1,"importBytes":2478,"outputBytes":1394,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:02 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":446572,"outputBytes":243298,"outputFiles":"dist/human.esm-nobundle.js"} 2021-05-30 23:20:37 STATE: Build for: browserNoBundle type: esm: {"imports":39,"importBytes":415475,"outputBytes":243743,"outputFiles":"dist/human.esm-nobundle.js"}
2021-05-30 18:44:02 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"} 2021-05-30 23:20:37 STATE: Build for: browserBundle type: tfjs: {"modules":1274,"moduleBytes":4114813,"imports":7,"importBytes":2478,"outputBytes":1111418,"outputFiles":"dist/tfjs.esm.js"}
2021-05-30 18:44:02 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1556596,"outputBytes":1351115,"outputFiles":"dist/human.js"} 2021-05-30 23:20:37 STATE: Build for: browserBundle type: iife: {"imports":39,"importBytes":1525499,"outputBytes":1351568,"outputFiles":"dist/human.js"}
2021-05-30 18:44:03 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1556596,"outputBytes":1351107,"outputFiles":"dist/human.esm.js"} 2021-05-30 23:20:38 STATE: Build for: browserBundle type: esm: {"imports":39,"importBytes":1525499,"outputBytes":1351560,"outputFiles":"dist/human.esm.js"}
2021-05-30 18:44:03 INFO:  Generate types: ["src/human.ts"] 2021-05-30 23:20:38 INFO:  Generate types: ["src/human.ts"]
2021-05-30 18:44:07 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"] 2021-05-30 23:20:43 INFO:  Update Change log: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-05-30 18:44:07 INFO:  Generate TypeDocs: ["src/human.ts"] 2021-05-30 23:20:43 INFO:  Generate TypeDocs: ["src/human.ts"]

181
src/draw/draw.ts
View File

@ -61,12 +61,12 @@ export const options: DrawOptions = {
drawLabels: <boolean>true, drawLabels: <boolean>true,
drawBoxes: <boolean>true, drawBoxes: <boolean>true,
drawPolygons: <boolean>true, drawPolygons: <boolean>true,
drawGaze: <boolean>false, drawGaze: <boolean>true,
fillPolygons: <boolean>false, fillPolygons: <boolean>false,
useDepth: <boolean>true, useDepth: <boolean>true,
useCurves: <boolean>false, useCurves: <boolean>false,
bufferedFactor: <number>2, bufferedFactor: <number>3,
bufferedOutput: <boolean>false, bufferedOutput: <boolean>true,
}; };
let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 }; let bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
@ -183,14 +183,14 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`); if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`);
// if (f.genderConfidence) labels.push(f.gender); // if (f.genderConfidence) labels.push(f.gender);
if (f.age) labels.push(`age: ${f.age || ''}`); if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`iris distance: ${f.iris}`); if (f.iris) labels.push(`distance: ${f.iris}`);
if (f.emotion && f.emotion.length > 0) { if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' ')); labels.push(emotion.join(' '));
} }
if (f.rotation && f.rotation.angle && f.rotation.gaze) { if (f.rotation && f.rotation.angle && f.rotation.gaze) {
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`); if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
if (f.rotation.gaze.angle) labels.push(`gaze: ${rad2deg(f.rotation.gaze.angle)}°`); if (f.rotation.gaze.bearing) labels.push(`gaze: ${rad2deg(f.rotation.gaze.bearing)}°`);
} }
if (labels.length === 0) labels.push('face'); if (labels.length === 0) labels.push('face');
ctx.fillStyle = localOptions.color; ctx.fillStyle = localOptions.color;
@ -245,10 +245,10 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.fill(); ctx.fill();
} }
} }
if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.angle) { if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.bearing) {
const leftGaze = [ const leftGaze = [
f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]), f.annotations['leftEyeIris'][0][0] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]), f.annotations['leftEyeIris'][0][1] - (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
]; ];
ctx.beginPath(); ctx.beginPath();
ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]); ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]);
@ -257,8 +257,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
ctx.stroke(); ctx.stroke();
const rightGaze = [ const rightGaze = [
f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[2]), f.annotations['rightEyeIris'][0][0] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.angle) * f.rotation.gaze.strength * f.box[3]), f.annotations['rightEyeIris'][0][1] - (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
]; ];
ctx.beginPath(); ctx.beginPath();
ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]); ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]);
@ -507,83 +507,108 @@ export async function person(inCanvas: HTMLCanvasElement, result: Array<Person>,
} }
} }
function calcBuffered(newResult, localOptions) { function calcBuffered(newResult: Result, localOptions) {
// each record is only updated using deep clone when number of detected record changes, otherwise it will converge by itself // each record is only updated using deep clone when number of detected record changes, otherwise it will converge by itself
// otherwise bufferedResult is a shallow clone of result plus updated local calculated values // otherwise bufferedResult is a shallow clone of result plus updated local calculated values
// thus mixing by-reference and by-value assignments to minimize memory operations // thus mixing by-reference and by-value assignments to minimize memory operations
// interpolate body results // interpolate body results
if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) bufferedResult.body = JSON.parse(JSON.stringify(newResult.body)); // deep clone once if (!bufferedResult.body || (newResult.body.length !== bufferedResult.body.length)) {
for (let i = 0; i < newResult.body.length; i++) { bufferedResult.body = JSON.parse(JSON.stringify(newResult.body)); // deep clone once
const box = newResult.body[i].box // update box } else {
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / localOptions.bufferedFactor) as [number, number, number, number]; for (let i = 0; i < newResult.body.length; i++) {
const boxRaw = newResult.body[i].boxRaw // update boxRaw const box = newResult.body[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / localOptions.bufferedFactor) as [number, number, number, number]; .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
const keypoints = newResult.body[i].keypoints // update keypoints const boxRaw = newResult.body[i].boxRaw // update boxRaw
.map((keypoint, j) => ({ .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / localOptions.bufferedFactor) as [number, number, number, number];
score: keypoint.score, const keypoints = newResult.body[i].keypoints // update keypoints
part: keypoint.part, .map((keypoint, j) => ({
position: { score: keypoint.score,
x: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / localOptions.bufferedFactor : keypoint.position.x, part: keypoint.part,
y: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / localOptions.bufferedFactor : keypoint.position.y, position: {
}, x: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.x + keypoint.position.x) / localOptions.bufferedFactor : keypoint.position.x,
})); y: bufferedResult.body[i].keypoints[j] ? ((localOptions.bufferedFactor - 1) * bufferedResult.body[i].keypoints[j].position.y + keypoint.position.y) / localOptions.bufferedFactor : keypoint.position.y,
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values },
}));
bufferedResult.body[i] = { ...newResult.body[i], box, boxRaw, keypoints }; // shallow clone plus updated values
}
} }
// interpolate hand results // interpolate hand results
if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand)); // deep clone once if (!bufferedResult.hand || (newResult.hand.length !== bufferedResult.hand.length)) {
for (let i = 0; i < newResult.hand.length; i++) { bufferedResult.hand = JSON.parse(JSON.stringify(newResult.hand)); // deep clone once
const box = newResult.hand[i].box // update box } else {
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / localOptions.bufferedFactor); for (let i = 0; i < newResult.hand.length; i++) {
const boxRaw = newResult.hand[i].boxRaw // update boxRaw const box = (newResult.hand[i].box// update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / localOptions.bufferedFactor); .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const landmarks = newResult.hand[i].landmarks // update landmarks const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
.map((landmark, j) => landmark .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].landmarks[j][k] + coord) / localOptions.bufferedFactor)); const landmarks = newResult.hand[i].landmarks // update landmarks
const keys = Object.keys(newResult.hand[i].annotations); // update annotations .map((landmark, j) => landmark
const annotations = []; .map((coord, k) => (((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].landmarks[j][k] + coord) / localOptions.bufferedFactor)) as [number, number, number]);
for (const key of keys) { const keys = Object.keys(newResult.hand[i].annotations); // update annotations
annotations[key] = newResult.hand[i].annotations[key] const annotations = {};
.map((val, j) => val for (const key of keys) {
.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / localOptions.bufferedFactor)); annotations[key] = newResult.hand[i].annotations[key]
.map((val, j) => val.map((coord, k) => ((localOptions.bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / localOptions.bufferedFactor));
}
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, landmarks, annotations }; // shallow clone plus updated values
} }
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, landmarks, annotations }; // shallow clone plus updated values
} }
// interpolate face results // interpolate face results
if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) bufferedResult.face = JSON.parse(JSON.stringify(newResult.face)); // deep clone once if (!bufferedResult.face || (newResult.face.length !== bufferedResult.face.length)) {
for (let i = 0; i < newResult.face.length; i++) { bufferedResult.face = JSON.parse(JSON.stringify(newResult.face)); // deep clone once
const box = newResult.face[i].box // update box } else {
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor); for (let i = 0; i < newResult.face.length; i++) {
const boxRaw = newResult.face[i].boxRaw // update boxRaw const box = (newResult.face[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor); .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
const matrix = newResult.face[i].rotation.matrix; const boxRaw = (newResult.face[i].boxRaw // update boxRaw
const angle = { .map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / localOptions.bufferedFactor)) as [number, number, number, number];
roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor, const matrix = newResult.face[i].rotation.matrix;
yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor, const angle = {
pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor, roll: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.roll + newResult.face[i].rotation.angle.roll) / localOptions.bufferedFactor,
}; yaw: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.yaw + newResult.face[i].rotation.angle.yaw) / localOptions.bufferedFactor,
const gaze = { pitch: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.angle.pitch + newResult.face[i].rotation.angle.pitch) / localOptions.bufferedFactor,
angle: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.angle + newResult.face[i].rotation.gaze.angle) / localOptions.bufferedFactor, };
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor, const gaze = {
}; bearing: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.bearing + newResult.face[i].rotation.gaze.bearing) / localOptions.bufferedFactor, // not correct due to wrap-around
const rotation = { angle, matrix, gaze }; /*
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values angle: Math.atan2( // average angle is calculated differently
Math.sin(bufferedResult.face[i].rotation.gaze.angle) + Math.sin(newResult.face[i].rotation.gaze.angle),
Math.cos(bufferedResult.face[i].rotation.gaze.angle) + Math.sin(newResult.face[i].rotation.gaze.angle),
),
*/
strength: ((localOptions.bufferedFactor - 1) * bufferedResult.face[i].rotation.gaze.strength + newResult.face[i].rotation.gaze.strength) / localOptions.bufferedFactor,
};
const rotation = { angle, matrix, gaze };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
}
}
// interpolate object detection results
if (!bufferedResult.object || (newResult.object.length !== bufferedResult.object.length)) {
bufferedResult.object = JSON.parse(JSON.stringify(newResult.object)); // deep clone once
} else {
for (let i = 0; i < newResult.object.length; i++) {
const box = newResult.object[i].box // update box
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / localOptions.bufferedFactor);
const boxRaw = newResult.object[i].boxRaw // update boxRaw
.map((b, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / localOptions.bufferedFactor);
bufferedResult.object[i] = { ...newResult.object[i], box, boxRaw }; // shallow clone plus updated values
}
} }
// interpolate person results // interpolate person results
const newPersons = newResult.persons; // trigger getter function const newPersons = newResult.persons; // trigger getter function
if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) bufferedResult.persons = JSON.parse(JSON.stringify(newPersons)); if (!bufferedResult.persons || (newPersons.length !== bufferedResult.persons.length)) {
for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow bufferedResult.persons = JSON.parse(JSON.stringify(newPersons));
bufferedResult.persons[i].box = newPersons[i].box } else {
.map((box, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / localOptions.bufferedFactor); for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow
bufferedResult.persons[i].box = (newPersons[i].box
.map((box, j) => ((localOptions.bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / localOptions.bufferedFactor)) as [number, number, number, number];
}
} }
// no buffering implemented for face, object, gesture
// bufferedResult.face = JSON.parse(JSON.stringify(newResult.face));
// bufferedResult.object = JSON.parse(JSON.stringify(newResult.object));
// bufferedResult.gesture = JSON.parse(JSON.stringify(newResult.gesture));
} }
export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasElement) { export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasElement) {
@ -597,12 +622,14 @@ export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptio
const localOptions = mergeDeep(options, drawOptions); const localOptions = mergeDeep(options, drawOptions);
if (!result || !inCanvas) return; if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return; if (!(inCanvas instanceof HTMLCanvasElement)) return;
if (localOptions.bufferedOutput) calcBuffered(result, localOptions); // do results interpolation
else bufferedResult = result; // just use results as-is if (!bufferedResult) bufferedResult = result; // first pass
face(inCanvas, bufferedResult.face, localOptions); // face does have buffering else if (localOptions.bufferedOutput) calcBuffered(result, localOptions); // do results interpolation
body(inCanvas, bufferedResult.body, localOptions); // use interpolated results if available else bufferedResult = result; // or just use results as-is
hand(inCanvas, bufferedResult.hand, localOptions); // use interpolated results if available face(inCanvas, bufferedResult.face, localOptions);
// person(inCanvas, bufferedResult.persons, localOptions); // use interpolated results if available body(inCanvas, bufferedResult.body, localOptions);
hand(inCanvas, bufferedResult.hand, localOptions);
object(inCanvas, bufferedResult.object, localOptions);
// person(inCanvas, bufferedResult.persons, localOptions);
gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering gesture(inCanvas, result.gesture, localOptions); // gestures do not have buffering
object(inCanvas, result.object, localOptions); // object detection does not have buffering
} }

14
src/face.ts
View File

@ -12,7 +12,7 @@ import { Face } from './result';
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const rad2deg = (theta) => (theta * 180) / Math.PI; const rad2deg = (theta) => (theta * 180) / Math.PI;
const calculateGaze = (mesh): { angle: number, strength: number } => { const calculateGaze = (mesh, box): { bearing: number, strength: number } => {
const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points const radians = (pt1, pt2) => Math.atan2(pt1[1] - pt2[1], pt1[0] - pt2[0]); // function to calculate angle between any two points
const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes const offsetIris = [0, -0.1]; // iris center may not align with average of eye extremes
@ -31,17 +31,17 @@ const calculateGaze = (mesh): { angle: number, strength: number } => {
(eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0], (eyeCenter[0] - irisCenter[0]) / eyeSize[0] - offsetIris[0],
eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1], eyeRatio * (irisCenter[1] - eyeCenter[1]) / eyeSize[1] - offsetIris[1],
]; ];
const vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences let vectorLength = Math.sqrt((eyeDiff[0] ** 2) + (eyeDiff[1] ** 2)); // vector length is a diagonal between two differences
vectorLength = Math.min(vectorLength, box[2] / 2, box[3] / 2); // limit strength to half of box size
const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments const vectorAngle = radians([0, 0], eyeDiff); // using eyeDiff instead eyeCenter/irisCenter combo due to manual adjustments
// vectorAngle right=0*pi, up=1*pi/2, left=1*pi, down=3*pi/2 return { bearing: vectorAngle, strength: vectorLength };
return { angle: vectorAngle, strength: vectorLength };
}; };
const calculateFaceAngle = (face, imageSize): { const calculateFaceAngle = (face, imageSize): {
angle: { pitch: number, yaw: number, roll: number }, angle: { pitch: number, yaw: number, roll: number },
matrix: [number, number, number, number, number, number, number, number, number], matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number }, gaze: { bearing: number, strength: number },
} => { } => {
// const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360); // const degrees = (theta) => Math.abs(((theta * 180) / Math.PI) % 360);
const normalize = (v) => { // normalize vector const normalize = (v) => { // normalize vector
@ -104,7 +104,7 @@ const calculateFaceAngle = (face, imageSize): {
// initialize gaze and mesh // initialize gaze and mesh
const mesh = face.meshRaw; const mesh = face.meshRaw;
if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { angle: 0, strength: 0 } }; if (!mesh || mesh.length < 300) return { angle: { pitch: 0, yaw: 0, roll: 0 }, matrix: [1, 0, 0, 0, 1, 0, 0, 0, 1], gaze: { bearing: 0, strength: 0 } };
const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5; const size = Math.max(face.boxRaw[2] * imageSize[0], face.boxRaw[3] * imageSize[1]) / 1.5;
// top, bottom, left, right // top, bottom, left, right
@ -132,7 +132,7 @@ const calculateFaceAngle = (face, imageSize): {
// const angle = meshToEulerAngle(mesh); // const angle = meshToEulerAngle(mesh);
// we have iris keypoints so we can calculate gaze direction // we have iris keypoints so we can calculate gaze direction
const gaze = mesh.length === 478 ? calculateGaze(mesh) : { angle: 0, strength: 0 }; const gaze = mesh.length === 478 ? calculateGaze(mesh, face.box) : { bearing: 0, strength: 0 };
return { angle, matrix, gaze }; return { angle, matrix, gaze };
}; };

14720
src/handpose/anchors.ts
View File

File diff suppressed because it is too large Load Diff

2
src/handpose/handpose.ts
View File

@ -35,7 +35,7 @@ export async function predict(input, config): Promise<Hand[]> {
} }
} }
const landmarks = predictions[i].landmarks as number[]; const landmarks = predictions[i].landmarks as unknown as Array<[number, number, number]>;
let box: [number, number, number, number] = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work let box: [number, number, number, number] = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work
let boxRaw: [number, number, number, number] = [0, 0, 0, 0]; let boxRaw: [number, number, number, number] = [0, 0, 0, 0];

8
src/result.ts
View File

@ -28,7 +28,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations * - rotation: face rotiation that contains both angles and matrix used for 3d transformations
* - angle: face angle as object with values for roll, yaw and pitch angles * - angle: face angle as object with values for roll, yaw and pitch angles
* - matrix: 3d transofrmation matrix as array of numeric values * - matrix: 3d transofrmation matrix as array of numeric values
* - gaze: gaze direction as object with values for agngle in radians and strengthss * - gaze: gaze direction as object with values for bearing in radians and relative strength
* - tensor: face tensor as Tensor object which contains detected face * - tensor: face tensor as Tensor object which contains detected face
*/ */
export interface Face { export interface Face {
@ -50,7 +50,7 @@ export interface Face {
rotation: { rotation: {
angle: { roll: number, yaw: number, pitch: number }, angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number], matrix: [number, number, number, number, number, number, number, number, number],
gaze: { angle: number, strength: number }, gaze: { bearing: number, strength: number },
} }
tensor: typeof Tensor, tensor: typeof Tensor,
} }
@ -97,8 +97,8 @@ export interface Hand {
confidence: number, confidence: number,
box: [number, number, number, number], box: [number, number, number, number],
boxRaw: [number, number, number, number], boxRaw: [number, number, number, number],
landmarks: number[], landmarks: Array<[number, number, number]>,
annotations: Record<string, Array<{ part: string, points: Array<[number, number, number]> }>>, annotations: Record<string, Array<[number, number, number]>>,
} }
/** Object results /** Object results

2
typedoc/assets/js/search.js
View File

File diff suppressed because one or more lines are too long

8
typedoc/interfaces/face.html
View File

@ -91,7 +91,7 @@
<li>rotation: face rotiation that contains both angles and matrix used for 3d transformations</li> <li>rotation: face rotiation that contains both angles and matrix used for 3d transformations</li>
<li>angle: face angle as object with values for roll, yaw and pitch angles</li> <li>angle: face angle as object with values for roll, yaw and pitch angles</li>
<li>matrix: 3d transofrmation matrix as array of numeric values</li> <li>matrix: 3d transofrmation matrix as array of numeric values</li>
<li>gaze: gaze direction as object with values for agngle in radians and strengthss</li> <li>gaze: gaze direction as object with values for bearing in radians and relative strength</li>
<li>tensor: face tensor as Tensor object which contains detected face</li> <li>tensor: face tensor as Tensor object which contains detected face</li>
</ul> </ul>
</div> </div>
@ -243,7 +243,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="rotation" class="tsd-anchor"></a> <a name="rotation" class="tsd-anchor"></a>
<h3>rotation</h3> <h3>rotation</h3>
<div class="tsd-signature tsd-kind-icon">rotation<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>pitch<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>roll<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>yaw<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>matrix<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol"> }</span></div> <div class="tsd-signature tsd-kind-icon">rotation<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>pitch<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>roll<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>yaw<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">; </span>matrix<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol"> }</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
<div class="tsd-type-declaration"> <div class="tsd-type-declaration">
@ -264,10 +264,10 @@
</ul> </ul>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></h5> <h5>gaze<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">{ </span>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">; </span>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol"> }</span></h5>
<ul class="tsd-parameters"> <ul class="tsd-parameters">
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>angle<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>bearing<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>
</li> </li>
<li class="tsd-parameter"> <li class="tsd-parameter">
<h5>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5> <h5>strength<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">number</span></h5>

4
typedoc/interfaces/hand.html
View File

@ -110,7 +110,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="annotations" class="tsd-anchor"></a> <a name="annotations" class="tsd-anchor"></a>
<h3>annotations</h3> <h3>annotations</h3>
<div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol">&lt;</span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">{ </span>part<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">; </span>points<span class="tsd-signature-symbol">: </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">&gt;</span></div> <div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol">&lt;</span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">&gt;</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
</section> </section>
@ -145,7 +145,7 @@
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface"> <section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
<a name="landmarks" class="tsd-anchor"></a> <a name="landmarks" class="tsd-anchor"></a>
<h3>landmarks</h3> <h3>landmarks</h3>
<div class="tsd-signature tsd-kind-icon">landmarks<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">[]</span></div> <div class="tsd-signature tsd-kind-icon">landmarks<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span></div>
<aside class="tsd-sources"> <aside class="tsd-sources">
</aside> </aside>
</section> </section>

11
types/result.d.ts vendored
View File

@ -26,7 +26,7 @@ import { Tensor } from '../dist/tfjs.esm.js';
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations * - rotation: face rotiation that contains both angles and matrix used for 3d transformations
* - angle: face angle as object with values for roll, yaw and pitch angles * - angle: face angle as object with values for roll, yaw and pitch angles
* - matrix: 3d transofrmation matrix as array of numeric values * - matrix: 3d transofrmation matrix as array of numeric values
* - gaze: gaze direction as object with values for agngle in radians and strengthss * - gaze: gaze direction as object with values for bearing in radians and relative strength
* - tensor: face tensor as Tensor object which contains detected face * - tensor: face tensor as Tensor object which contains detected face
*/ */
export interface Face { export interface Face {
@ -59,7 +59,7 @@ export interface Face {
}; };
matrix: [number, number, number, number, number, number, number, number, number]; matrix: [number, number, number, number, number, number, number, number, number];
gaze: { gaze: {
angle: number; bearing: number;
strength: number; strength: number;
}; };
}; };
@ -114,11 +114,8 @@ export interface Hand {
confidence: number; confidence: number;
box: [number, number, number, number]; box: [number, number, number, number];
boxRaw: [number, number, number, number]; boxRaw: [number, number, number, number];
landmarks: number[]; landmarks: Array<[number, number, number]>;
annotations: Record<string, Array<{ annotations: Record<string, Array<[number, number, number]>>;
part: string;
points: Array<[number, number, number]>;
}>>;
} }
/** Object results /** Object results
* *