From 250207e67e2c4260b73902583f39a90e254fdf07 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 5 Mar 2021 11:43:50 -0500 Subject: [PATCH] update human.draw helper methods --- README.md | 38 ++++- demo/browser.js | 89 ++++------ src/draw.ts | 436 ++++++++++++++++++++++++++++++++++++++++++++++++ src/human.ts | 3 + wiki | 2 +- 5 files changed, 511 insertions(+), 57 deletions(-) create mode 100644 src/draw.ts diff --git a/README.md b/README.md index 7c23cce3..85f04618 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,14 @@ Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows an - Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends - NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu* -Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) for processing of live WebCam video or static images +Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) for processing of live WebCam video or static images +Live demo uses `WASM` backend for faster startup, but results are slower than when using `WebGL` backend
## Project pages -- [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) +- [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) - [**Code Repository**](https://github.com/vladmandic/human) - [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human) - [**Issues Tracker**](https://github.com/vladmandic/human/issues) @@ -102,5 +103,38 @@ As presented in the demo application... ![Example Using WebCam](assets/screenshot-webcam.jpg) +

+ +Example simple app that uses Human to process video input and +draw output on screen using internal draw helper functions + +```js +import Human from '@vladmandic/human'; + +// create instance of human with simple configuration using default values +const config = { backend: 'wasm' }; +const human = new Human(config); + +function detectVideo() { + // select input HTMLVideoElement and output HTMLCanvasElement from page + const inputVideo = document.getElementById('video-id'); + const outputCanvas = document.getElementById('canvas-id'); + // perform processing using default configuration + human.detect(inputVideo).then((result) => { + // result object will contain detected details as well as the processed canvas itself + // first draw processed frame on canvas + human.draw.canvas(result.canvas, outputCanvas); + // then draw results on the same canvas + human.draw.face(outputCanvas, result.face); + human.draw.body(outputCanvas, result.body); + human.draw.hand(outputCanvas, result.hand); + human.draw.gesture(outputCanvas, result.gesture); + // loop immediate to next frame + requestAnimationFrame(detectVideo); + }); +} + +detectVideo(); +```
diff --git a/demo/browser.js b/demo/browser.js index 4829406b..da05564a 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -4,11 +4,10 @@ import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human -import draw from './draw.js'; import Menu from './menu.js'; import GLBench from './gl-bench.js'; -const userConfig = {}; // add any user configuration overrides +const userConfig = { backend: 'wasm' }; // add any user configuration overrides /* const userConfig = { @@ -27,40 +26,31 @@ const human = new Human(userConfig); // ui options const ui = { - baseColor: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel baseBackground: 'rgba(50, 50, 50, 1)', // 'grey' - baseLabel: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel - baseFontProto: 'small-caps {size} "Segoe UI"', - baseLineWidth: 12, - crop: true, - columns: 2, - busy: false, - facing: true, - useWorker: false, + crop: true, // video mode crop to size or leave full frame + columns: 2, // when processing sample images create this many columns + facing: true, // camera facing front or back + useWorker: false, // use web workers for processing worker: 'worker.js', samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'], compare: '../assets/sample-me.jpg', - drawLabels: true, - drawBoxes: true, - drawPoints: true, - drawPolygons: true, - fillPolygons: false, - useDepth: true, - console: true, - maxFPSframes: 10, - modelsPreload: true, - menuWidth: 0, - menuHeight: 0, - camera: {}, - detectFPS: [], - drawFPS: [], - buffered: false, - drawWarmup: false, - drawThread: null, - detectThread: null, - framesDraw: 0, - framesDetect: 0, - bench: false, + console: true, // log messages to browser console + maxFPSframes: 10, // keep fps history for how many frames + modelsPreload: true, // preload human models on startup + busy: false, // internal camera busy flag + menuWidth: 0, // internal + menuHeight: 0, // internal + camera: {}, // internal, holds details of webcam details + detectFPS: [], // internal, holds fps values for detection performance + drawFPS: [], // internal, holds fps values for draw performance + buffered: false, // experimental, should output be buffered between frames + drawWarmup: false, // debug only, should warmup image processing be displayed on startup + drawThread: null, // perform draw operations in a separate thread + detectThread: null, // perform detect operations in a separate thread + framesDraw: 0, // internal, statistics on frames drawn + framesDetect: 0, // internal, statistics on frames detected + bench: false, // show gl fps benchmark window + lastFrame: 0, // time of last frame processing }; // global variables @@ -90,7 +80,8 @@ function log(...msg) { function status(msg) { // eslint-disable-next-line no-console - document.getElementById('status').innerText = msg; + const div = document.getElementById('status'); + if (div) div.innerText = msg; } let original; @@ -139,10 +130,10 @@ async function drawResults(input) { } // draw all results - await draw.face(result.face, canvas, ui, human.facemesh.triangulation); - await draw.body(result.body, canvas, ui); - await draw.hand(result.hand, canvas, ui); - await draw.gesture(result.gesture, canvas, ui); + await human.draw.face(canvas, result.face); + await human.draw.body(canvas, result.body); + await human.draw.hand(canvas, result.hand); + await human.draw.gesture(canvas, result.gesture); await calcSimmilariry(result); // update log @@ -230,9 +221,6 @@ async function setupCamera() { ui.menuWidth.input.setAttribute('value', video.width); ui.menuHeight.input.setAttribute('value', video.height); // silly font resizing for paint-on-canvas since viewport can be zoomed - const size = Math.trunc(window.devicePixelRatio * (8 + (4 * canvas.width / window.innerWidth))); - ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`); - ui.baseLineHeight = size + 2; if (live) video.play(); // eslint-disable-next-line no-use-before-define if (live && !ui.detectThread) runHumanDetect(video, canvas); @@ -404,9 +392,6 @@ async function detectVideo() { async function detectSampleImages() { document.getElementById('play').style.display = 'none'; userConfig.videoOptimized = false; - const size = Math.trunc(window.devicePixelRatio * (12 + (4 * ui.columns))); - ui.baseFont = ui.baseFontProto.replace(/{size}/, `${size}px`); - ui.baseLineHeight = size + 2; document.getElementById('canvas').style.display = 'none'; document.getElementById('samples-container').style.display = 'block'; log('Running detection of sample images'); @@ -439,12 +424,12 @@ function setupMenu() { setupCamera(); }); menu.display.addHTML('
'); - menu.display.addBool('use 3D depth', ui, 'useDepth'); - menu.display.addBool('print labels', ui, 'drawLabels'); - menu.display.addBool('draw boxes', ui, 'drawBoxes'); - menu.display.addBool('draw polygons', ui, 'drawPolygons'); - menu.display.addBool('Fill Polygons', ui, 'fillPolygons'); - menu.display.addBool('draw points', ui, 'drawPoints'); + menu.display.addBool('use 3D depth', human.draw.options, 'useDepth'); + menu.display.addBool('print labels', human.draw.options, 'drawLabels'); + menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes'); + menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons'); + menu.display.addBool('Fill Polygons', human.draw.options, 'fillPolygons'); + menu.display.addBool('draw points', human.draw.options, 'drawPoints'); menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] }); menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val); @@ -541,10 +526,7 @@ async function drawWarmup(res) { canvas.height = res.canvas.height; const ctx = canvas.getContext('2d'); ctx.drawImage(res.canvas, 0, 0, res.canvas.width, res.canvas.height, 0, 0, canvas.width, canvas.height); - await draw.face(res.face, canvas, ui, human.facemesh.triangulation); - await draw.body(res.body, canvas, ui); - await draw.hand(res.hand, canvas, ui); - await draw.gesture(res.gesture, canvas, ui); + await human.draw.all(canvas, res); } async function main() { @@ -561,7 +543,6 @@ async function main() { if (!ui.useWorker) { status('initializing'); const res = await human.warmup(userConfig); // this is not required, just pre-warms all models for faster initial inference - ui.baseFont = ui.baseFontProto.replace(/{size}/, '16px'); if (res && res.canvas && ui.drawWarmup) await drawWarmup(res); } status('human: ready'); diff --git a/src/draw.ts b/src/draw.ts new file mode 100644 index 00000000..5cdcc84d --- /dev/null +++ b/src/draw.ts @@ -0,0 +1,436 @@ +import { TRI468 as triangulation } from './blazeface/coords'; + +export const options = { + color: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel + labelColor: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel + shadowColor: 'black', + font: 'small-caps 16px "Segoe UI"', + lineHeight: 20, + lineWidth: 6, + pointSize: 2, + roundRect: 8, + drawLabels: true, + drawBoxes: true, + drawPoints: false, + drawPolygons: true, + fillPolygons: true, + useDepth: true, + bufferedOutput: false, +}; + +function point(ctx, x, y) { + ctx.fillStyle = options.color; + ctx.beginPath(); + ctx.arc(x, y, options.pointSize, 0, 2 * Math.PI); + ctx.fill(); +} + +function rect(ctx, x, y, width, height) { + if (options.roundRect && options.roundRect > 0) { + ctx.lineWidth = options.lineWidth; + ctx.beginPath(); + ctx.moveTo(x + options.roundRect, y); + ctx.lineTo(x + width - options.roundRect, y); + ctx.quadraticCurveTo(x + width, y, x + width, y + options.roundRect); + ctx.lineTo(x + width, y + height - options.roundRect); + ctx.quadraticCurveTo(x + width, y + height, x + width - options.roundRect, y + height); + ctx.lineTo(x + options.roundRect, y + height); + ctx.quadraticCurveTo(x, y + height, x, y + height - options.roundRect); + ctx.lineTo(x, y + options.roundRect); + ctx.quadraticCurveTo(x, y, x + options.roundRect, y); + ctx.closePath(); + ctx.stroke(); + } else { + rect(ctx, x, y, width, height); + } +} + +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +function curve(ctx, points = []) { + if (points.length < 2) return; + ctx.lineWidth = options.lineWidth; + ctx.beginPath(); + ctx.moveTo(points[0][0], points[0][1]); + for (let i = 0; i < points.length - 1; i++) { + const xMid = (points[i][0] + points[i + 1][0]) / 2; + const yMid = (points[i][1] + points[i + 1][1]) / 2; + const cpX1 = (xMid + points[i][0]) / 2; + const cpX2 = (xMid + points[i + 1][1]) / 2; + ctx.quadraticCurveTo(cpX1, points[i][1], xMid, yMid); + ctx.quadraticCurveTo(cpX2, points[i + 1][1], points[i + 1][0], points[i + 1][0]); + } + ctx.strokeStyle = options.color; + ctx.stroke(); +} + +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +function bezier(ctx, points) { + const tension = 0; // tension at 0 will be straight line + const factor = 1; // factor is normally 1, but changing the value can control the smoothness too + if (points.length < 2) return; + ctx.lineWidth = options.lineWidth; + ctx.strokeStyle = options.color; + ctx.fillStyle = options.color; + ctx.beginPath(); + ctx.moveTo(points[0][0], points[0][1]); + let dx1 = 0; + let dy1 = 0; + let preP = points[0]; + for (let i = 1; i < points.length; i++) { + const curP = points[i]; + const nexP = points[i + 1]; + const m = nexP ? (nexP[1] - preP[1]) / (nexP[0] - preP[0]) : 0; + const dx2 = nexP ? (nexP[0] - curP[0]) * -factor : 0; + const dy2 = nexP ? dx2 * m * tension : 0; + ctx.bezierCurveTo(preP[0] - dx1, preP[1] - dy1, curP[0] + dx2, curP[1] + dy2, curP[0], curP[1]); + dx1 = dx2; + dy1 = dy2; + preP = curP; + } + ctx.stroke(); +} + +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +function spline(ctx, points) { + const tension = 0.8; + if (points.length < 2) return; + const va = (arr, i, j) => [arr[2 * j] - arr[2 * i], arr[2 * j + 1] - arr[2 * i + 1]]; + const distance = (arr, i, j) => Math.sqrt(((arr[2 * i] - arr[2 * j]) ** 2) + ((arr[2 * i + 1] - arr[2 * j + 1]) ** 2)); + // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars + const ctlpts = (x1, y1, x2, y2, x3, y3) => { + // eslint-disable-next-line prefer-rest-params + const v = va(arguments, 0, 2); + // eslint-disable-next-line prefer-rest-params + const d01 = distance(arguments, 0, 1); + // eslint-disable-next-line prefer-rest-params + const d12 = distance(arguments, 1, 2); + const d012 = d01 + d12; + return [ + x2 - v[0] * tension * d01 / d012, y2 - v[1] * tension * d01 / d012, + x2 + v[0] * tension * d12 / d012, y2 + v[1] * tension * d12 / d012, + ]; + }; + const pts: any[] = []; + for (const pt of points) { + pts.push(pt[0]); + pts.push(pt[1]); + } + let cps = []; + for (let i = 0; i < pts.length - 2; i += 1) { + // @ts-ignore + cps = cps.concat(ctlpts(pts[2 * i + 0], pts[2 * i + 1], pts[2 * i + 2], pts[2 * i + 3], pts[2 * i + 4], pts[2 * i + 5])); + } + ctx.lineWidth = options.lineWidth; + ctx.strokeStyle = options.color; + if (points.length === 2) { + ctx.beginPath(); + ctx.moveTo(pts[0], pts[1]); + ctx.lineTo(pts[2], pts[3]); + } else { + ctx.beginPath(); + ctx.moveTo(pts[0], pts[1]); + // first segment is a quadratic + ctx.quadraticCurveTo(cps[0], cps[1], pts[2], pts[3]); + // for all middle points, connect with bezier + let i; + for (i = 2; i < ((pts.length / 2) - 1); i += 1) { + ctx.bezierCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], cps[(2 * (i - 1)) * 2], cps[(2 * (i - 1)) * 2 + 1], pts[i * 2], pts[i * 2 + 1]); + } + // last segment is a quadratic + ctx.quadraticCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], pts[i * 2], pts[i * 2 + 1]); + } + ctx.stroke(); +} + +export async function gesture(inCanvas, result) { + if (!result || !inCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement)) return; + const ctx = inCanvas.getContext('2d'); + if (!ctx) return; + ctx.font = options.font; + ctx.fillStyle = options.color; + let i = 1; + for (let j = 0; j < result.length; j++) { + let where:any[] = []; + let what:any[] = []; + [where, what] = Object.entries(result[j]); + if ((what.length > 1) && (what[1].length > 0)) { + const person = where[1] > 0 ? `#${where[1]}` : ''; + const label = `${where[0]} ${person}: ${what[1]}`; + if (options.shadowColor && options.shadowColor !== '') { + ctx.fillStyle = options.shadowColor; + ctx.fillText(label, 8, 2 + (i * options.lineHeight)); + } + ctx.fillStyle = options.labelColor; + ctx.fillText(label, 6, 0 + (i * options.lineHeight)); + i += 1; + } + } +} + +export async function face(inCanvas, result) { + if (!result || !inCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement)) return; + const ctx = inCanvas.getContext('2d'); + if (!ctx) return; + for (const f of result) { + ctx.font = options.font; + ctx.strokeStyle = options.color; + ctx.fillStyle = options.color; + ctx.lineWidth = options.lineWidth; + ctx.beginPath(); + if (options.drawBoxes) { + rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); + } + // silly hack since fillText does not suport new line + const labels:string[] = []; + labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`); + if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`); + // if (f.genderConfidence) labels.push(f.gender); + if (f.age) labels.push(`age: ${f.age || ''}`); + if (f.iris) labels.push(`iris distance: ${f.iris}`); + if (f.emotion && f.emotion.length > 0) { + const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); + labels.push(emotion.join(' ')); + } + if (labels.length === 0) labels.push('face'); + ctx.fillStyle = options.color; + for (let i = labels.length - 1; i >= 0; i--) { + const x = Math.max(f.box[0], 0); + const y = i * options.lineHeight + f.box[1]; + if (options.shadowColor && options.shadowColor !== '') { + ctx.fillStyle = options.shadowColor; + ctx.fillText(labels[i], x + 5, y + 16); + } + ctx.fillStyle = options.labelColor; + ctx.fillText(labels[i], x + 4, y + 15); + } + ctx.fillStyle = options.color; + ctx.stroke(); + ctx.lineWidth = 1; + if (f.mesh) { + if (options.drawPoints) { + for (const pt of f.mesh) { + ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color; + point(ctx, pt[0], pt[1]); + } + } + if (options.drawPolygons) { + for (let i = 0; i < triangulation.length / 3; i++) { + const points = [ + triangulation[i * 3 + 0], + triangulation[i * 3 + 1], + triangulation[i * 3 + 2], + ].map((index) => f.mesh[index]); + const path = new Path2D(); + path.moveTo(points[0][0], points[0][1]); + for (const pt of points) { + path.lineTo(pt[0], pt[1]); + } + path.closePath(); + ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color; + ctx.stroke(path); + if (options.fillPolygons) { + ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color; + ctx.fill(path); + } + } + // iris: array[center, left, top, right, bottom] + if (f.annotations && f.annotations.leftEyeIris) { + ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color; + ctx.beginPath(); + const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2; + const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2; + ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + ctx.stroke(); + if (options.fillPolygons) { + ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color; + ctx.fill(); + } + } + if (f.annotations && f.annotations.rightEyeIris) { + ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color; + ctx.beginPath(); + const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2; + const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2; + ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + ctx.stroke(); + if (options.fillPolygons) { + ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color; + ctx.fill(); + } + } + } + } + } +} + +const lastDrawnPose:any[] = []; +export async function body(inCanvas, result) { + if (!result || !inCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement)) return; + const ctx = inCanvas.getContext('2d'); + if (!ctx) return; + ctx.lineJoin = 'round'; + for (let i = 0; i < result.length; i++) { + // result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5); + if (!lastDrawnPose[i] && options.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; + ctx.strokeStyle = options.color; + ctx.font = options.font; + ctx.lineWidth = options.lineWidth; + if (options.drawPoints) { + for (let pt = 0; pt < result[i].keypoints.length; pt++) { + ctx.fillStyle = options.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : options.color; + if (options.drawLabels) { + ctx.fillText(`${result[i].keypoints[pt].part}`, result[i].keypoints[pt][0] + 4, result[i].keypoints[pt][1] + 4); + } + ctx.beginPath(); + if (options.bufferedOutput) { + lastDrawnPose[i].keypoints[pt][0] = (lastDrawnPose[i].keypoints[pt][0] + result[i].keypoints[pt][0]) / 2; + lastDrawnPose[i].keypoints[pt][1] = (lastDrawnPose[i].keypoints[pt][1] + result[i].keypoints[pt][1]) / 2; + point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1]); + } else { + point(ctx, result[i].keypoints[pt][0], result[i].keypoints[pt][1]); + } + ctx.fill(); + } + } + if (options.drawPolygons) { + const path = new Path2D(); + let root; + let part; + // torso + root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); + if (root) { + path.moveTo(root.position.x, root.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightHip'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftHip'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); + if (part) path.lineTo(part.position.x, part.position.y); + } + // leg left + root = result[i].keypoints.find((a) => a.part === 'leftHip'); + if (root) { + path.moveTo(root.position.x, root.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftKnee'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftHeel'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftFoot'); + if (part) path.lineTo(part.position.x, part.position.y); + } + // leg right + root = result[i].keypoints.find((a) => a.part === 'rightHip'); + if (root) { + path.moveTo(root.position.x, root.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightKnee'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightHeel'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightFoot'); + if (part) path.lineTo(part.position.x, part.position.y); + } + // arm left + root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); + if (root) { + path.moveTo(root.position.x, root.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftElbow'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftWrist'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'leftPalm'); + if (part) path.lineTo(part.position.x, part.position.y); + } + // arm right + root = result[i].keypoints.find((a) => a.part === 'rightShoulder'); + if (root) { + path.moveTo(root.position.x, root.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightElbow'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightWrist'); + if (part) path.lineTo(part.position.x, part.position.y); + part = result[i].keypoints.find((a) => a.part === 'rightPalm'); + if (part) path.lineTo(part.position.x, part.position.y); + } + // draw all + ctx.stroke(path); + } + } +} + +export async function hand(inCanvas, result) { + if (!result || !inCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement)) return; + const ctx = inCanvas.getContext('2d'); + if (!ctx) return; + ctx.lineJoin = 'round'; + for (const h of result) { + ctx.font = options.font; + ctx.lineWidth = options.lineWidth; + if (options.drawBoxes) { + ctx.lineWidth = options.lineWidth; + ctx.beginPath(); + ctx.strokeStyle = options.color; + ctx.fillStyle = options.color; + rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); + if (options.shadowColor && options.shadowColor !== '') { + ctx.fillStyle = options.shadowColor; + ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + options.lineHeight, h.box[2]); + } + ctx.fillStyle = options.labelColor; + ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + options.lineHeight, h.box[2]); + ctx.stroke(); + } + if (options.drawPoints) { + if (h.landmarks && h.landmarks.length > 0) { + for (const pt of h.landmarks) { + ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color; + point(ctx, pt[0], pt[1]); + } + } + } + if (options.drawPolygons) { + const addPart = (part) => { + if (!part) return; + for (let i = 0; i < part.length; i++) { + ctx.lineWidth = options.lineWidth; + ctx.beginPath(); + ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : options.color; + ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]); + ctx.lineTo(part[i][0], part[i][1]); + ctx.stroke(); + } + }; + addPart(h.annotations.indexFinger); + addPart(h.annotations.middleFinger); + addPart(h.annotations.ringFinger); + addPart(h.annotations.pinky); + addPart(h.annotations.thumb); + // addPart(hand.annotations.palmBase); + } + } +} + +export async function canvas(inCanvas, outCanvas) { + if (!inCanvas || !outCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement) || !(outCanvas instanceof HTMLCanvasElement)) return; + const outCtx = inCanvas.getContext('2d'); + outCtx?.drawImage(inCanvas, 0, 0); +} + +export async function all(inCanvas, result) { + if (!result || !inCanvas) return; + if (!(inCanvas instanceof HTMLCanvasElement)) return; + face(inCanvas, result.face); + body(inCanvas, result.body); + hand(inCanvas, result.hand); + gesture(inCanvas, result.gesture); +} diff --git a/src/human.ts b/src/human.ts index 68961212..0f0b3516 100644 --- a/src/human.ts +++ b/src/human.ts @@ -16,6 +16,7 @@ import * as profile from './profile'; import * as config from '../config'; import * as sample from './sample'; import * as app from '../package.json'; +import * as draw from './draw'; // helper function: gets elapsed time on both browser and nodejs const now = () => { @@ -40,6 +41,7 @@ function mergeDeep(...objects) { class Human { tf: any; + draw: any; package: any; version: string; config: any; @@ -62,6 +64,7 @@ class Human { constructor(userConfig = {}) { this.tf = tf; + this.draw = draw; this.package = app; this.version = app.version; this.config = mergeDeep(config.default, userConfig); diff --git a/wiki b/wiki index b624d76a..9173ba06 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit b624d76a06647aa4542a65b7454cf9a8ac57c1c3 +Subproject commit 9173ba06bb5f5aa361ea391ea88b6aaf0eb34006