ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update human.draw helper methods

pull/91/head
Vladimir Mandic 2021-03-05 11:43:50 -05:00
parent 32247c1b06
commit 35d720dbd3
24 changed files with 1442 additions and 1213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
README.md
View File

@ -13,13 +13,14 @@ Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows an
- Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends - Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends
- NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu* - NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu*
Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) for processing of live WebCam video or static images Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) for processing of live WebCam video or static images
Live demo uses `WASM` backend for faster startup, but results are slower than when using `WebGL` backend
<br> <br>
## Project pages ## Project pages
- [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) - [**Live Demo**](https://vladmandic.github.io/human/demo/index.html)
- [**Code Repository**](https://github.com/vladmandic/human) - [**Code Repository**](https://github.com/vladmandic/human)
- [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human) - [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human)
- [**Issues Tracker**](https://github.com/vladmandic/human/issues) - [**Issues Tracker**](https://github.com/vladmandic/human/issues)
@ -102,5 +103,38 @@ As presented in the demo application...
![Example Using WebCam](assets/screenshot-webcam.jpg) ![Example Using WebCam](assets/screenshot-webcam.jpg)
<br><hr><br>
Example simple app that uses Human to process video input and
draw output on screen using internal draw helper functions
```js
import Human from '@vladmandic/human';
// create instance of human with simple configuration using default values
const config = { backend: 'wasm' };
const human = new Human(config);
function detectVideo() {
// select input HTMLVideoElement and output HTMLCanvasElement from page
const inputVideo = document.getElementById('video-id');
const outputCanvas = document.getElementById('canvas-id');
// perform processing using default configuration
human.detect(inputVideo).then((result) => {
// result object will contain detected details as well as the processed canvas itself
// first draw processed frame on canvas
human.draw.canvas(result.canvas, outputCanvas);
// then draw results on the same canvas
human.draw.face(outputCanvas, result.face);
human.draw.body(outputCanvas, result.body);
human.draw.hand(outputCanvas, result.hand);
human.draw.gesture(outputCanvas, result.gesture);
// loop immediate to next frame
requestAnimationFrame(detectVideo);
});
}
detectVideo();
```
<br> <br>

89
demo/browser.js
View File

@ -4,11 +4,10 @@
import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
import draw from './draw.js';
import Menu from './menu.js'; import Menu from './menu.js';
import GLBench from './gl-bench.js'; import GLBench from './gl-bench.js';
const userConfig = {}; // add any user configuration overrides const userConfig = { backend: 'wasm' }; // add any user configuration overrides
/* /*
const userConfig = { const userConfig = {
@ -27,40 +26,31 @@ const human = new Human(userConfig);
// ui options // ui options
const ui = { const ui = {
baseColor: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel
baseBackground: 'rgba(50, 50, 50, 1)', // 'grey' baseBackground: 'rgba(50, 50, 50, 1)', // 'grey'
baseLabel: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel crop: true, // video mode crop to size or leave full frame
baseFontProto: 'small-caps {size} "Segoe UI"', columns: 2, // when processing sample images create this many columns
baseLineWidth: 12, facing: true, // camera facing front or back
crop: true, useWorker: false, // use web workers for processing
columns: 2,
busy: false,
facing: true,
useWorker: false,
worker: 'worker.js', worker: 'worker.js',
samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'], samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'],
compare: '../assets/sample-me.jpg', compare: '../assets/sample-me.jpg',
drawLabels: true, console: true, // log messages to browser console
drawBoxes: true, maxFPSframes: 10, // keep fps history for how many frames
drawPoints: true, modelsPreload: true, // preload human models on startup
drawPolygons: true, busy: false, // internal camera busy flag
fillPolygons: false, menuWidth: 0, // internal
useDepth: true, menuHeight: 0, // internal
console: true, camera: {}, // internal, holds details of webcam details
maxFPSframes: 10, detectFPS: [], // internal, holds fps values for detection performance
modelsPreload: true, drawFPS: [], // internal, holds fps values for draw performance
menuWidth: 0, buffered: false, // experimental, should output be buffered between frames
menuHeight: 0, drawWarmup: false, // debug only, should warmup image processing be displayed on startup
camera: {}, drawThread: null, // perform draw operations in a separate thread
detectFPS: [], detectThread: null, // perform detect operations in a separate thread
drawFPS: [], framesDraw: 0, // internal, statistics on frames drawn
buffered: false, framesDetect: 0, // internal, statistics on frames detected
drawWarmup: false, bench: false, // show gl fps benchmark window
drawThread: null, lastFrame: 0, // time of last frame processing
detectThread: null,
framesDraw: 0,
framesDetect: 0,
bench: false,
}; };
// global variables // global variables
@ -90,7 +80,8 @@ function log(...msg) {
function status(msg) { function status(msg) {
// eslint-disable-next-line no-console // eslint-disable-next-line no-console
document.getElementById('status').innerText = msg; const div = document.getElementById('status');
if (div) div.innerText = msg;
} }
let original; let original;
@ -139,10 +130,10 @@ async function drawResults(input) {
} }
// draw all results // draw all results
await draw.face(result.face, canvas, ui, human.facemesh.triangulation); await human.draw.face(canvas, result.face);
await draw.body(result.body, canvas, ui); await human.draw.body(canvas, result.body);
await draw.hand(result.hand, canvas, ui); await human.draw.hand(canvas, result.hand);
await draw.gesture(result.gesture, canvas, ui); await human.draw.gesture(canvas, result.gesture);
await calcSimmilariry(result); await calcSimmilariry(result);
// update log // update log
@ -230,9 +221,6 @@ async function setupCamera() {
ui.menuWidth.input.setAttribute('value', video.width); ui.menuWidth.input.setAttribute('value', video.width);
ui.menuHeight.input.setAttribute('value', video.height); ui.menuHeight.input.setAttribute('value', video.height);
// silly font resizing for paint-on-canvas since viewport can be zoomed // silly font resizing for paint-on-canvas since viewport can be zoomed
const size = Math.trunc(window.devicePixelRatio * (8 + (4 * canvas.width / window.innerWidth)));
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`);
ui.baseLineHeight = size + 2;
if (live) video.play(); if (live) video.play();
// eslint-disable-next-line no-use-before-define // eslint-disable-next-line no-use-before-define
if (live && !ui.detectThread) runHumanDetect(video, canvas); if (live && !ui.detectThread) runHumanDetect(video, canvas);
@ -404,9 +392,6 @@ async function detectVideo() {
async function detectSampleImages() { async function detectSampleImages() {
document.getElementById('play').style.display = 'none'; document.getElementById('play').style.display = 'none';
userConfig.videoOptimized = false; userConfig.videoOptimized = false;
const size = Math.trunc(window.devicePixelRatio * (12 + (4 * ui.columns)));
ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`);
ui.baseLineHeight = size + 2;
document.getElementById('canvas').style.display = 'none'; document.getElementById('canvas').style.display = 'none';
document.getElementById('samples-container').style.display = 'block'; document.getElementById('samples-container').style.display = 'block';
log('Running detection of sample images'); log('Running detection of sample images');
@ -439,12 +424,12 @@ function setupMenu() {
setupCamera(); setupCamera();
}); });
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">'); menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
menu.display.addBool('use 3D depth', ui, 'useDepth'); menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
menu.display.addBool('print labels', ui, 'drawLabels'); menu.display.addBool('print labels', human.draw.options, 'drawLabels');
menu.display.addBool('draw boxes', ui, 'drawBoxes'); menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
menu.display.addBool('draw polygons', ui, 'drawPolygons'); menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons');
menu.display.addBool('Fill Polygons', ui, 'fillPolygons'); menu.display.addBool('Fill Polygons', human.draw.options, 'fillPolygons');
menu.display.addBool('draw points', ui, 'drawPoints'); menu.display.addBool('draw points', human.draw.options, 'drawPoints');
menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] }); menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] });
menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val); menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val);
@ -541,10 +526,7 @@ async function drawWarmup(res) {
canvas.height = res.canvas.height; canvas.height = res.canvas.height;
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.drawImage(res.canvas, 0, 0, res.canvas.width, res.canvas.height, 0, 0, canvas.width, canvas.height); ctx.drawImage(res.canvas, 0, 0, res.canvas.width, res.canvas.height, 0, 0, canvas.width, canvas.height);
await draw.face(res.face, canvas, ui, human.facemesh.triangulation); await human.draw.all(canvas, res);
await draw.body(res.body, canvas, ui);
await draw.hand(res.hand, canvas, ui);
await draw.gesture(res.gesture, canvas, ui);
} }
async function main() { async function main() {
@ -561,7 +543,6 @@ async function main() {
if (!ui.useWorker) { if (!ui.useWorker) {
status('initializing'); status('initializing');
const res = await human.warmup(userConfig); // this is not required, just pre-warms all models for faster initial inference const res = await human.warmup(userConfig); // this is not required, just pre-warms all models for faster initial inference
ui.baseFont = ui.baseFontProto.replace(/{size}/, '16px');
if (res && res.canvas && ui.drawWarmup) await drawWarmup(res); if (res && res.canvas && ui.drawWarmup) await drawWarmup(res);
} }
status('human: ready'); status('human: ready');

286
demo/draw.js
View File

@ -1,286 +0,0 @@
async function drawPoint(canvas, x = 0, y = 0, radius = 0, color = 'black', label) {
const ctx = canvas.getContext('2d');
ctx.fillStyle = color;
ctx.beginPath();
ctx.arc(x, y, radius, 0, 2 * Math.PI);
ctx.fill();
if (label) ctx.fillText(label, x + 4, y + 4);
}
async function drawGesture(result, canvas, ui) {
if (!result) return;
const ctx = canvas.getContext('2d');
ctx.font = ui.baseFont;
ctx.fillStyle = ui.baseLabel;
let i = 1;
for (let gesture = 0; gesture < result.length; gesture++) {
const [where, what] = Object.entries(result[gesture]);
if ((what.length > 1) && (what[1].length > 0)) {
const person = where[1] > 0 ? `#${where[1]}` : '';
const label = `${where[0]} ${person}: ${what[1]}`;
ctx.fillStyle = 'black';
ctx.fillText(label, 8, 2 + (i * ui.baseLineHeight));
// ctx.fillText(label, 151, i * 16 + 101);
ctx.fillStyle = ui.baseLabel;
// ctx.fillText(label, 150, i * 16 + 100);
ctx.fillText(label, 6, 0 + (i * ui.baseLineHeight));
i += 1;
}
}
}
async function drawFace(result, canvas, ui, triangulation) {
if (!result) return;
const ctx = canvas.getContext('2d');
for (const face of result) {
ctx.font = ui.baseFont;
ctx.strokeStyle = ui.baseColor;
ctx.fillStyle = ui.baseColor;
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
if (ui.drawBoxes) {
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
}
// silly hack since fillText does not suport new line
const labels = [];
labels.push(`detect confidence: ${Math.trunc(100 * face.confidence)}%`);
if (face.genderConfidence) labels.push(`${face.gender || ''} ${Math.trunc(100 * face.genderConfidence)}% confident`);
// if (face.genderConfidence) labels.push(face.gender);
if (face.age) labels.push(`age: ${face.age || ''}`);
if (face.iris) labels.push(`iris distance: ${face.iris}`);
if (face.emotion && face.emotion.length > 0) {
const emotion = face.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' '));
}
if (labels.length === 0) labels.push('face');
ctx.fillStyle = ui.baseLabel;
for (let i = labels.length - 1; i >= 0; i--) {
ctx.fillStyle = 'black';
const x = Math.max(face.box[0], 0);
const y = i * ui.baseLineHeight + face.box[1];
ctx.fillText(labels[i], x + 5, y + 16);
// ctx.fillText(labels[i], 151, i * 16 + 28);
ctx.fillStyle = ui.baseLabel;
ctx.fillText(labels[i], x + 4, y + 15);
// ctx.fillText(labels[i], 150, i * 16 + 27);
}
ctx.fillStyle = ui.baseColor;
ctx.stroke();
ctx.lineWidth = 1;
if (face.mesh) {
if (ui.drawPoints) {
for (const point of face.mesh) {
ctx.fillStyle = ui.useDepth ? `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)` : ui.baseColor;
ctx.beginPath();
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
ctx.fill();
}
}
if (ui.drawPolygons) {
for (let i = 0; i < triangulation.length / 3; i++) {
const points = [
triangulation[i * 3 + 0],
triangulation[i * 3 + 1],
triangulation[i * 3 + 2],
].map((index) => face.mesh[index]);
const path = new Path2D();
path.moveTo(points[0][0], points[0][1]);
for (const point of points) {
path.lineTo(point[0], point[1]);
}
path.closePath();
ctx.strokeStyle = ui.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : ui.baseColor;
ctx.stroke(path);
if (ui.fillPolygons) {
ctx.fillStyle = ui.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : ui.baseColor;
ctx.fill(path);
}
}
// iris: array[center, left, top, right, bottom]
if (face.annotations && face.annotations.leftEyeIris) {
ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.leftEyeIris[4][1] - face.annotations.leftEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.leftEyeIris[0][0], face.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui.fillPolygons) {
ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
ctx.fill();
}
}
if (face.annotations && face.annotations.rightEyeIris) {
ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.rightEyeIris[0][0], face.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui.fillPolygons) {
ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
ctx.fill();
}
}
}
}
}
}
const lastDrawnPose = [];
async function drawBody(result, canvas, ui) {
if (!result) return;
const ctx = canvas.getContext('2d');
ctx.lineJoin = 'round';
for (let i = 0; i < result.length; i++) {
// result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5);
if (!lastDrawnPose[i] && ui.buffered) lastDrawnPose[i] = { ...result[i] };
ctx.strokeStyle = ui.baseColor;
ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
if (ui.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = ui.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : ui.baseColor;
if (ui.drawLabels) {
ctx.fillText(`${result[i].keypoints[pt].part}`, result[i].keypoints[pt].position.x + 4, result[i].keypoints[pt].position.y + 4);
}
ctx.beginPath();
if (ui.buffered) {
lastDrawnPose[i].keypoints[pt].position.x = (lastDrawnPose[i].keypoints[pt].position.x + result[i].keypoints[pt].position.x) / 2;
lastDrawnPose[i].keypoints[pt].position.y = (lastDrawnPose[i].keypoints[pt].position.y + result[i].keypoints[pt].position.y) / 2;
ctx.arc(lastDrawnPose[i].keypoints[pt].position.x, lastDrawnPose[i].keypoints[pt].position.y, 2, 0, 2 * Math.PI);
} else {
ctx.arc(result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 2, 0, 2 * Math.PI);
}
ctx.fill();
}
}
if (ui.drawPolygons) {
const path = new Path2D();
let root;
let part;
// torso
root = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) path.lineTo(part.position.x, part.position.y);
}
// leg left
root = result[i].keypoints.find((a) => a.part === 'leftHip');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part) path.lineTo(part.position.x, part.position.y);
}
// leg right
root = result[i].keypoints.find((a) => a.part === 'rightHip');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part) path.lineTo(part.position.x, part.position.y);
}
// arm left
root = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part) path.lineTo(part.position.x, part.position.y);
}
// arm right
root = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part) path.lineTo(part.position.x, part.position.y);
}
// draw all
ctx.stroke(path);
}
}
}
async function drawHand(result, canvas, ui) {
if (!result) return;
const ctx = canvas.getContext('2d');
ctx.lineJoin = 'round';
for (const hand of result) {
ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
if (ui.drawBoxes) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
ctx.strokeStyle = ui.baseColor;
ctx.fillStyle = ui.baseColor;
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
ctx.fillStyle = 'black';
ctx.fillText('hand', hand.box[0] + 3, 1 + hand.box[1] + ui.baseLineHeight, hand.box[2]);
ctx.fillStyle = ui.baseLabel;
ctx.fillText('hand', hand.box[0] + 2, 0 + hand.box[1] + ui.baseLineHeight, hand.box[2]);
ctx.stroke();
}
if (ui.drawPoints) {
if (hand.landmarks && hand.landmarks.length > 0) {
for (const point of hand.landmarks) {
ctx.fillStyle = ui.useDepth ? `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)` : ui.baseColor;
ctx.beginPath();
ctx.arc(point[0], point[1], 2, 0, 2 * Math.PI);
ctx.fill();
}
}
}
if (ui.drawPolygons) {
const addPart = (part) => {
if (!part) return;
for (let i = 0; i < part.length; i++) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
ctx.strokeStyle = ui.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : ui.baseColor;
ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]);
ctx.lineTo(part[i][0], part[i][1]);
ctx.stroke();
}
};
addPart(hand.annotations.indexFinger);
addPart(hand.annotations.middleFinger);
addPart(hand.annotations.ringFinger);
addPart(hand.annotations.pinky);
addPart(hand.annotations.thumb);
// addPart(hand.annotations.palmBase);
}
}
}
// eslint-disable-next-line import/prefer-default-export
export default {
face: drawFace,
body: drawBody,
hand: drawHand,
gesture: drawGesture,
point: drawPoint,
};

520
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

23
dist/demo-browser-index.json vendored
View File

@ -1,11 +1,7 @@
{ {
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1342423, "bytes": 1350737,
"imports": []
},
"demo/draw.js": {
"bytes": 12536,
"imports": [] "imports": []
}, },
"demo/menu.js": { "demo/menu.js": {
@ -17,16 +13,12 @@
"imports": [] "imports": []
}, },
"demo/browser.js": { "demo/browser.js": {
"bytes": 28109, "bytes": 28114,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js", "path": "dist/human.esm.js",
"kind": "import-statement" "kind": "import-statement"
}, },
{
"path": "demo/draw.js",
"kind": "import-statement"
},
{ {
"path": "demo/menu.js", "path": "demo/menu.js",
"kind": "import-statement" "kind": "import-statement"
@ -43,7 +35,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 2052296 "bytes": 2058908
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
@ -51,10 +43,7 @@
"entryPoint": "demo/browser.js", "entryPoint": "demo/browser.js",
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1334941 "bytesInOutput": 1343243
},
"demo/draw.js": {
"bytesInOutput": 7178
}, },
"demo/menu.js": { "demo/menu.js": {
"bytesInOutput": 10696 "bytesInOutput": 10696
@ -63,10 +52,10 @@
"bytesInOutput": 6759 "bytesInOutput": 6759
}, },
"demo/browser.js": { "demo/browser.js": {
"bytesInOutput": 18268 "bytesInOutput": 17687
} }
}, },
"bytes": 1385227 "bytes": 1385770
} }
} }
} }

18
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

528
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

30
dist/human.esm.json vendored
View File

@ -388,7 +388,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 2879, "bytes": 3290,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -445,8 +445,17 @@
"bytes": 2594, "bytes": 2594,
"imports": [] "imports": []
}, },
"src/draw.ts": {
"bytes": 18011,
"imports": [
{
"path": "src/blazeface/coords.ts",
"kind": "import-statement"
}
]
},
"src/human.ts": { "src/human.ts": {
"bytes": 21043, "bytes": 21110,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -519,6 +528,10 @@
{ {
"path": "package.json", "path": "package.json",
"kind": "import-statement" "kind": "import-statement"
},
{
"path": "src/draw.ts",
"kind": "import-statement"
} }
] ]
} }
@ -528,7 +541,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1949289 "bytes": 1976930
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -547,7 +560,7 @@
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056695 "bytesInOutput": 1056713
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -568,7 +581,7 @@
"bytesInOutput": 5054 "bytesInOutput": 5054
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11312 "bytesInOutput": 11325
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1576 "bytesInOutput": 1576
@ -634,7 +647,7 @@
"bytesInOutput": 126985 "bytesInOutput": 126985
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytesInOutput": 1153 "bytesInOutput": 1167
}, },
"src/blazepose/annotations.ts": { "src/blazepose/annotations.ts": {
"bytesInOutput": 860 "bytesInOutput": 860
@ -656,9 +669,12 @@
}, },
"package.json": { "package.json": {
"bytesInOutput": 2596 "bytesInOutput": 2596
},
"src/draw.ts": {
"bytesInOutput": 8269
} }
}, },
"bytes": 1342423 "bytes": 1350737
} }
} }
} }

30
dist/human.iife.json vendored
View File

@ -388,7 +388,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 2879, "bytes": 3290,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -445,8 +445,17 @@
"bytes": 2594, "bytes": 2594,
"imports": [] "imports": []
}, },
"src/draw.ts": {
"bytes": 18011,
"imports": [
{
"path": "src/blazeface/coords.ts",
"kind": "import-statement"
}
]
},
"src/human.ts": { "src/human.ts": {
"bytes": 21043, "bytes": 21110,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -519,6 +528,10 @@
{ {
"path": "package.json", "path": "package.json",
"kind": "import-statement" "kind": "import-statement"
},
{
"path": "src/draw.ts",
"kind": "import-statement"
} }
] ]
} }
@ -528,7 +541,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1949300 "bytes": 1976941
}, },
"dist/human.ts": { "dist/human.ts": {
"imports": [], "imports": [],
@ -542,13 +555,13 @@
"bytesInOutput": 1690 "bytesInOutput": 1690
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11348 "bytesInOutput": 11361
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1056695 "bytesInOutput": 1056713
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1053 "bytesInOutput": 1053
@ -632,7 +645,7 @@
"bytesInOutput": 126985 "bytesInOutput": 126985
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytesInOutput": 1153 "bytesInOutput": 1167
}, },
"src/blazepose/annotations.ts": { "src/blazepose/annotations.ts": {
"bytesInOutput": 860 "bytesInOutput": 860
@ -654,9 +667,12 @@
}, },
"package.json": { "package.json": {
"bytesInOutput": 2596 "bytesInOutput": 2596
},
"src/draw.ts": {
"bytesInOutput": 8269
} }
}, },
"bytes": 1342465 "bytes": 1350779
} }
} }
} }

18
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

18
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

28
dist/human.node.json vendored
View File

@ -388,7 +388,7 @@
"imports": [] "imports": []
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytes": 2879, "bytes": 3290,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -445,8 +445,17 @@
"bytes": 2594, "bytes": 2594,
"imports": [] "imports": []
}, },
"src/draw.ts": {
"bytes": 18011,
"imports": [
{
"path": "src/blazeface/coords.ts",
"kind": "import-statement"
}
]
},
"src/human.ts": { "src/human.ts": {
"bytes": 21043, "bytes": 21110,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -519,6 +528,10 @@
{ {
"path": "package.json", "path": "package.json",
"kind": "import-statement" "kind": "import-statement"
},
{
"path": "src/draw.ts",
"kind": "import-statement"
} }
] ]
} }
@ -528,7 +541,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 716046 "bytes": 743687
}, },
"dist/human.node-gpu.js": { "dist/human.node-gpu.js": {
"imports": [], "imports": [],
@ -545,7 +558,7 @@
"bytesInOutput": 1677 "bytesInOutput": 1677
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 11320 "bytesInOutput": 11333
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 251 "bytesInOutput": 251
@ -632,7 +645,7 @@
"bytesInOutput": 126985 "bytesInOutput": 126985
}, },
"src/blazepose/blazepose.ts": { "src/blazepose/blazepose.ts": {
"bytesInOutput": 1179 "bytesInOutput": 1198
}, },
"src/blazepose/annotations.ts": { "src/blazepose/annotations.ts": {
"bytesInOutput": 860 "bytesInOutput": 860
@ -654,9 +667,12 @@
}, },
"package.json": { "package.json": {
"bytesInOutput": 2593 "bytesInOutput": 2593
},
"src/draw.ts": {
"bytesInOutput": 8180
} }
}, },
"bytes": 279967 "bytes": 288179
} }
} }
} }

528
dist/human.ts vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.ts.map vendored
View File

File diff suppressed because one or more lines are too long

436
src/draw.ts Normal file
View File

@ -0,0 +1,436 @@
import { TRI468 as triangulation } from './blazeface/coords';
export const options = {
color: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel
labelColor: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel
shadowColor: 'black',
font: 'small-caps 16px "Segoe UI"',
lineHeight: 20,
lineWidth: 6,
pointSize: 2,
roundRect: 8,
drawLabels: true,
drawBoxes: true,
drawPoints: false,
drawPolygons: true,
fillPolygons: true,
useDepth: true,
bufferedOutput: false,
};
function point(ctx, x, y) {
ctx.fillStyle = options.color;
ctx.beginPath();
ctx.arc(x, y, options.pointSize, 0, 2 * Math.PI);
ctx.fill();
}
function rect(ctx, x, y, width, height) {
if (options.roundRect && options.roundRect > 0) {
ctx.lineWidth = options.lineWidth;
ctx.beginPath();
ctx.moveTo(x + options.roundRect, y);
ctx.lineTo(x + width - options.roundRect, y);
ctx.quadraticCurveTo(x + width, y, x + width, y + options.roundRect);
ctx.lineTo(x + width, y + height - options.roundRect);
ctx.quadraticCurveTo(x + width, y + height, x + width - options.roundRect, y + height);
ctx.lineTo(x + options.roundRect, y + height);
ctx.quadraticCurveTo(x, y + height, x, y + height - options.roundRect);
ctx.lineTo(x, y + options.roundRect);
ctx.quadraticCurveTo(x, y, x + options.roundRect, y);
ctx.closePath();
ctx.stroke();
} else {
rect(ctx, x, y, width, height);
}
}
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
function curve(ctx, points = []) {
if (points.length < 2) return;
ctx.lineWidth = options.lineWidth;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (let i = 0; i < points.length - 1; i++) {
const xMid = (points[i][0] + points[i + 1][0]) / 2;
const yMid = (points[i][1] + points[i + 1][1]) / 2;
const cpX1 = (xMid + points[i][0]) / 2;
const cpX2 = (xMid + points[i + 1][1]) / 2;
ctx.quadraticCurveTo(cpX1, points[i][1], xMid, yMid);
ctx.quadraticCurveTo(cpX2, points[i + 1][1], points[i + 1][0], points[i + 1][0]);
}
ctx.strokeStyle = options.color;
ctx.stroke();
}
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
function bezier(ctx, points) {
const tension = 0; // tension at 0 will be straight line
const factor = 1; // factor is normally 1, but changing the value can control the smoothness too
if (points.length < 2) return;
ctx.lineWidth = options.lineWidth;
ctx.strokeStyle = options.color;
ctx.fillStyle = options.color;
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
let dx1 = 0;
let dy1 = 0;
let preP = points[0];
for (let i = 1; i < points.length; i++) {
const curP = points[i];
const nexP = points[i + 1];
const m = nexP ? (nexP[1] - preP[1]) / (nexP[0] - preP[0]) : 0;
const dx2 = nexP ? (nexP[0] - curP[0]) * -factor : 0;
const dy2 = nexP ? dx2 * m * tension : 0;
ctx.bezierCurveTo(preP[0] - dx1, preP[1] - dy1, curP[0] + dx2, curP[1] + dy2, curP[0], curP[1]);
dx1 = dx2;
dy1 = dy2;
preP = curP;
}
ctx.stroke();
}
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
function spline(ctx, points) {
const tension = 0.8;
if (points.length < 2) return;
const va = (arr, i, j) => [arr[2 * j] - arr[2 * i], arr[2 * j + 1] - arr[2 * i + 1]];
const distance = (arr, i, j) => Math.sqrt(((arr[2 * i] - arr[2 * j]) ** 2) + ((arr[2 * i + 1] - arr[2 * j + 1]) ** 2));
// eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
const ctlpts = (x1, y1, x2, y2, x3, y3) => {
// eslint-disable-next-line prefer-rest-params
const v = va(arguments, 0, 2);
// eslint-disable-next-line prefer-rest-params
const d01 = distance(arguments, 0, 1);
// eslint-disable-next-line prefer-rest-params
const d12 = distance(arguments, 1, 2);
const d012 = d01 + d12;
return [
x2 - v[0] * tension * d01 / d012, y2 - v[1] * tension * d01 / d012,
x2 + v[0] * tension * d12 / d012, y2 + v[1] * tension * d12 / d012,
];
};
const pts: any[] = [];
for (const pt of points) {
pts.push(pt[0]);
pts.push(pt[1]);
}
let cps = [];
for (let i = 0; i < pts.length - 2; i += 1) {
// @ts-ignore
cps = cps.concat(ctlpts(pts[2 * i + 0], pts[2 * i + 1], pts[2 * i + 2], pts[2 * i + 3], pts[2 * i + 4], pts[2 * i + 5]));
}
ctx.lineWidth = options.lineWidth;
ctx.strokeStyle = options.color;
if (points.length === 2) {
ctx.beginPath();
ctx.moveTo(pts[0], pts[1]);
ctx.lineTo(pts[2], pts[3]);
} else {
ctx.beginPath();
ctx.moveTo(pts[0], pts[1]);
// first segment is a quadratic
ctx.quadraticCurveTo(cps[0], cps[1], pts[2], pts[3]);
// for all middle points, connect with bezier
let i;
for (i = 2; i < ((pts.length / 2) - 1); i += 1) {
ctx.bezierCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], cps[(2 * (i - 1)) * 2], cps[(2 * (i - 1)) * 2 + 1], pts[i * 2], pts[i * 2 + 1]);
}
// last segment is a quadratic
ctx.quadraticCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], pts[i * 2], pts[i * 2 + 1]);
}
ctx.stroke();
}
export async function gesture(inCanvas, result) {
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
const ctx = inCanvas.getContext('2d');
if (!ctx) return;
ctx.font = options.font;
ctx.fillStyle = options.color;
let i = 1;
for (let j = 0; j < result.length; j++) {
let where:any[] = [];
let what:any[] = [];
[where, what] = Object.entries(result[j]);
if ((what.length > 1) && (what[1].length > 0)) {
const person = where[1] > 0 ? `#${where[1]}` : '';
const label = `${where[0]} ${person}: ${what[1]}`;
if (options.shadowColor && options.shadowColor !== '') {
ctx.fillStyle = options.shadowColor;
ctx.fillText(label, 8, 2 + (i * options.lineHeight));
}
ctx.fillStyle = options.labelColor;
ctx.fillText(label, 6, 0 + (i * options.lineHeight));
i += 1;
}
}
}
export async function face(inCanvas, result) {
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
const ctx = inCanvas.getContext('2d');
if (!ctx) return;
for (const f of result) {
ctx.font = options.font;
ctx.strokeStyle = options.color;
ctx.fillStyle = options.color;
ctx.lineWidth = options.lineWidth;
ctx.beginPath();
if (options.drawBoxes) {
rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]);
}
// silly hack since fillText does not suport new line
const labels:string[] = [];
labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`);
if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`);
// if (f.genderConfidence) labels.push(f.gender);
if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`iris distance: ${f.iris}`);
if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
labels.push(emotion.join(' '));
}
if (labels.length === 0) labels.push('face');
ctx.fillStyle = options.color;
for (let i = labels.length - 1; i >= 0; i--) {
const x = Math.max(f.box[0], 0);
const y = i * options.lineHeight + f.box[1];
if (options.shadowColor && options.shadowColor !== '') {
ctx.fillStyle = options.shadowColor;
ctx.fillText(labels[i], x + 5, y + 16);
}
ctx.fillStyle = options.labelColor;
ctx.fillText(labels[i], x + 4, y + 15);
}
ctx.fillStyle = options.color;
ctx.stroke();
ctx.lineWidth = 1;
if (f.mesh) {
if (options.drawPoints) {
for (const pt of f.mesh) {
ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color;
point(ctx, pt[0], pt[1]);
}
}
if (options.drawPolygons) {
for (let i = 0; i < triangulation.length / 3; i++) {
const points = [
triangulation[i * 3 + 0],
triangulation[i * 3 + 1],
triangulation[i * 3 + 2],
].map((index) => f.mesh[index]);
const path = new Path2D();
path.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
path.lineTo(pt[0], pt[1]);
}
path.closePath();
ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color;
ctx.stroke(path);
if (options.fillPolygons) {
ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color;
ctx.fill(path);
}
}
// iris: array[center, left, top, right, bottom]
if (f.annotations && f.annotations.leftEyeIris) {
ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color;
ctx.beginPath();
const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2;
const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2;
ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (options.fillPolygons) {
ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color;
ctx.fill();
}
}
if (f.annotations && f.annotations.rightEyeIris) {
ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color;
ctx.beginPath();
const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2;
const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2;
ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (options.fillPolygons) {
ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color;
ctx.fill();
}
}
}
}
}
}
const lastDrawnPose:any[] = [];
export async function body(inCanvas, result) {
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
const ctx = inCanvas.getContext('2d');
if (!ctx) return;
ctx.lineJoin = 'round';
for (let i = 0; i < result.length; i++) {
// result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5);
if (!lastDrawnPose[i] && options.bufferedOutput) lastDrawnPose[i] = { ...result[i] };
ctx.strokeStyle = options.color;
ctx.font = options.font;
ctx.lineWidth = options.lineWidth;
if (options.drawPoints) {
for (let pt = 0; pt < result[i].keypoints.length; pt++) {
ctx.fillStyle = options.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : options.color;
if (options.drawLabels) {
ctx.fillText(`${result[i].keypoints[pt].part}`, result[i].keypoints[pt][0] + 4, result[i].keypoints[pt][1] + 4);
}
ctx.beginPath();
if (options.bufferedOutput) {
lastDrawnPose[i].keypoints[pt][0] = (lastDrawnPose[i].keypoints[pt][0] + result[i].keypoints[pt][0]) / 2;
lastDrawnPose[i].keypoints[pt][1] = (lastDrawnPose[i].keypoints[pt][1] + result[i].keypoints[pt][1]) / 2;
point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1]);
} else {
point(ctx, result[i].keypoints[pt][0], result[i].keypoints[pt][1]);
}
ctx.fill();
}
}
if (options.drawPolygons) {
const path = new Path2D();
let root;
let part;
// torso
root = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightHip');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftHip');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (part) path.lineTo(part.position.x, part.position.y);
}
// leg left
root = result[i].keypoints.find((a) => a.part === 'leftHip');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftKnee');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftAnkle');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftHeel');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
if (part) path.lineTo(part.position.x, part.position.y);
}
// leg right
root = result[i].keypoints.find((a) => a.part === 'rightHip');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightKnee');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightAnkle');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightHeel');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
if (part) path.lineTo(part.position.x, part.position.y);
}
// arm left
root = result[i].keypoints.find((a) => a.part === 'leftShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftElbow');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftWrist');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
if (part) path.lineTo(part.position.x, part.position.y);
}
// arm right
root = result[i].keypoints.find((a) => a.part === 'rightShoulder');
if (root) {
path.moveTo(root.position.x, root.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightElbow');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightWrist');
if (part) path.lineTo(part.position.x, part.position.y);
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
if (part) path.lineTo(part.position.x, part.position.y);
}
// draw all
ctx.stroke(path);
}
}
}
export async function hand(inCanvas, result) {
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
const ctx = inCanvas.getContext('2d');
if (!ctx) return;
ctx.lineJoin = 'round';
for (const h of result) {
ctx.font = options.font;
ctx.lineWidth = options.lineWidth;
if (options.drawBoxes) {
ctx.lineWidth = options.lineWidth;
ctx.beginPath();
ctx.strokeStyle = options.color;
ctx.fillStyle = options.color;
rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]);
if (options.shadowColor && options.shadowColor !== '') {
ctx.fillStyle = options.shadowColor;
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + options.lineHeight, h.box[2]);
}
ctx.fillStyle = options.labelColor;
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + options.lineHeight, h.box[2]);
ctx.stroke();
}
if (options.drawPoints) {
if (h.landmarks && h.landmarks.length > 0) {
for (const pt of h.landmarks) {
ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color;
point(ctx, pt[0], pt[1]);
}
}
}
if (options.drawPolygons) {
const addPart = (part) => {
if (!part) return;
for (let i = 0; i < part.length; i++) {
ctx.lineWidth = options.lineWidth;
ctx.beginPath();
ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : options.color;
ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]);
ctx.lineTo(part[i][0], part[i][1]);
ctx.stroke();
}
};
addPart(h.annotations.indexFinger);
addPart(h.annotations.middleFinger);
addPart(h.annotations.ringFinger);
addPart(h.annotations.pinky);
addPart(h.annotations.thumb);
// addPart(hand.annotations.palmBase);
}
}
}
export async function canvas(inCanvas, outCanvas) {
if (!inCanvas || !outCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement) || !(outCanvas instanceof HTMLCanvasElement)) return;
const outCtx = inCanvas.getContext('2d');
outCtx?.drawImage(inCanvas, 0, 0);
}
export async function all(inCanvas, result) {
if (!result || !inCanvas) return;
if (!(inCanvas instanceof HTMLCanvasElement)) return;
face(inCanvas, result.face);
body(inCanvas, result.body);
hand(inCanvas, result.hand);
gesture(inCanvas, result.gesture);
}

3
src/human.ts
View File

@ -16,6 +16,7 @@ import * as profile from './profile';
import * as config from '../config'; import * as config from '../config';
import * as sample from './sample'; import * as sample from './sample';
import * as app from '../package.json'; import * as app from '../package.json';
import * as draw from './draw';
// helper function: gets elapsed time on both browser and nodejs // helper function: gets elapsed time on both browser and nodejs
const now = () => { const now = () => {
@ -40,6 +41,7 @@ function mergeDeep(...objects) {
class Human { class Human {
tf: any; tf: any;
draw: any;
package: any; package: any;
version: string; version: string;
config: any; config: any;
@ -62,6 +64,7 @@ class Human {
constructor(userConfig = {}) { constructor(userConfig = {}) {
this.tf = tf; this.tf = tf;
this.draw = draw;
this.package = app; this.package = app;
this.version = app.version; this.version = app.version;
this.config = mergeDeep(config.default, userConfig); this.config = mergeDeep(config.default, userConfig);

23
types/draw.d.ts vendored Normal file
View File

@ -0,0 +1,23 @@
export declare const options: {
color: string;
labelColor: string;
shadowColor: string;
font: string;
lineHeight: number;
lineWidth: number;
pointSize: number;
roundRect: number;
drawLabels: boolean;
drawBoxes: boolean;
drawPoints: boolean;
drawPolygons: boolean;
fillPolygons: boolean;
useDepth: boolean;
bufferedOutput: boolean;
};
export declare function gesture(inCanvas: any, result: any): Promise<void>;
export declare function face(inCanvas: any, result: any): Promise<void>;
export declare function body(inCanvas: any, result: any): Promise<void>;
export declare function hand(inCanvas: any, result: any): Promise<void>;
export declare function canvas(inCanvas: any, outCanvas: any): Promise<void>;
export declare function all(inCanvas: any, result: any): Promise<void>;

1
types/human.d.ts vendored
View File

@ -1,5 +1,6 @@
declare class Human { declare class Human {
tf: any; tf: any;
draw: any;
package: any; package: any;
version: string; version: string;
config: any; config: any;

2
wiki

@ -1 +1 @@
Subproject commit b624d76a06647aa4542a65b7454cf9a8ac57c1c3 Subproject commit 9173ba06bb5f5aa361ea391ea88b6aaf0eb34006