import config from '../config'; import { TRI468 as triangulation } from './blazeface/coords'; export const options = { color: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel labelColor: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel shadowColor: 'black', font: 'small-caps 16px "Segoe UI"', lineHeight: 20, lineWidth: 6, pointSize: 2, roundRect: 8, drawPoints: false, drawLabels: true, drawBoxes: true, drawPolygons: true, fillPolygons: false, useDepth: true, bufferedOutput: false, }; function point(ctx, x, y) { ctx.fillStyle = options.color; ctx.beginPath(); ctx.arc(x, y, options.pointSize, 0, 2 * Math.PI); ctx.fill(); } function rect(ctx, x, y, width, height) { if (options.roundRect && options.roundRect > 0) { ctx.lineWidth = options.lineWidth; ctx.beginPath(); ctx.moveTo(x + options.roundRect, y); ctx.lineTo(x + width - options.roundRect, y); ctx.quadraticCurveTo(x + width, y, x + width, y + options.roundRect); ctx.lineTo(x + width, y + height - options.roundRect); ctx.quadraticCurveTo(x + width, y + height, x + width - options.roundRect, y + height); ctx.lineTo(x + options.roundRect, y + height); ctx.quadraticCurveTo(x, y + height, x, y + height - options.roundRect); ctx.lineTo(x, y + options.roundRect); ctx.quadraticCurveTo(x, y, x + options.roundRect, y); ctx.closePath(); ctx.stroke(); } else { rect(ctx, x, y, width, height); } } // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function lines(ctx, points) { ctx.beginPath(); const path = new Path2D(); path.moveTo(points[0][0], points[0][1]); for (const pt of points) { path.lineTo(pt[0], parseInt(pt[1])); } ctx.stroke(path); if (options.fillPolygons) { ctx.closePath(); ctx.fill(path); } } // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function curve(ctx, points = []) { if (points.length < 2) return; ctx.lineWidth = options.lineWidth; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); for (let i = 0; i < points.length - 1; i++) { const xMid = (points[i][0] + points[i + 1][0]) / 2; const yMid = (points[i][1] + points[i + 1][1]) / 2; const cpX1 = (xMid + points[i][0]) / 2; const cpX2 = (xMid + points[i + 1][1]) / 2; ctx.quadraticCurveTo(cpX1, points[i][1], xMid, yMid); ctx.quadraticCurveTo(cpX2, points[i + 1][1], points[i + 1][0], points[i + 1][0]); } ctx.strokeStyle = options.color; ctx.stroke(); } // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function bezier(ctx, points) { const tension = 0; // tension at 0 will be straight line const factor = 1; // factor is normally 1, but changing the value can control the smoothness too if (points.length < 2) return; ctx.lineWidth = options.lineWidth; ctx.strokeStyle = options.color; ctx.fillStyle = options.color; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); let dx1 = 0; let dy1 = 0; let preP = points[0]; for (let i = 1; i < points.length; i++) { const curP = points[i]; const nexP = points[i + 1]; const m = nexP ? (nexP[1] - preP[1]) / (nexP[0] - preP[0]) : 0; const dx2 = nexP ? (nexP[0] - curP[0]) * -factor : 0; const dy2 = nexP ? dx2 * m * tension : 0; ctx.bezierCurveTo(preP[0] - dx1, preP[1] - dy1, curP[0] + dx2, curP[1] + dy2, curP[0], curP[1]); dx1 = dx2; dy1 = dy2; preP = curP; } ctx.stroke(); } // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars function spline(ctx, points) { const tension = 0.8; if (points.length < 2) return; const va = (arr, i, j) => [arr[2 * j] - arr[2 * i], arr[2 * j + 1] - arr[2 * i + 1]]; const distance = (arr, i, j) => Math.sqrt(((arr[2 * i] - arr[2 * j]) ** 2) + ((arr[2 * i + 1] - arr[2 * j + 1]) ** 2)); // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars const ctlpts = (x1, y1, x2, y2, x3, y3) => { // eslint-disable-next-line prefer-rest-params const v = va(arguments, 0, 2); // eslint-disable-next-line prefer-rest-params const d01 = distance(arguments, 0, 1); // eslint-disable-next-line prefer-rest-params const d12 = distance(arguments, 1, 2); const d012 = d01 + d12; return [ x2 - v[0] * tension * d01 / d012, y2 - v[1] * tension * d01 / d012, x2 + v[0] * tension * d12 / d012, y2 + v[1] * tension * d12 / d012, ]; }; const pts: any[] = []; for (const pt of points) { pts.push(pt[0]); pts.push(pt[1]); } let cps = []; for (let i = 0; i < pts.length - 2; i += 1) { // @ts-ignore cps = cps.concat(ctlpts(pts[2 * i + 0], pts[2 * i + 1], pts[2 * i + 2], pts[2 * i + 3], pts[2 * i + 4], pts[2 * i + 5])); } ctx.lineWidth = options.lineWidth; ctx.strokeStyle = options.color; if (points.length === 2) { ctx.beginPath(); ctx.moveTo(pts[0], pts[1]); ctx.lineTo(pts[2], pts[3]); } else { ctx.beginPath(); ctx.moveTo(pts[0], pts[1]); // first segment is a quadratic ctx.quadraticCurveTo(cps[0], cps[1], pts[2], pts[3]); // for all middle points, connect with bezier let i; for (i = 2; i < ((pts.length / 2) - 1); i += 1) { ctx.bezierCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], cps[(2 * (i - 1)) * 2], cps[(2 * (i - 1)) * 2 + 1], pts[i * 2], pts[i * 2 + 1]); } // last segment is a quadratic ctx.quadraticCurveTo(cps[(2 * (i - 1) - 1) * 2], cps[(2 * (i - 1) - 1) * 2 + 1], pts[i * 2], pts[i * 2 + 1]); } ctx.stroke(); } export async function gesture(inCanvas, result) { if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; ctx.font = options.font; ctx.fillStyle = options.color; let i = 1; for (let j = 0; j < result.length; j++) { let where:any[] = []; let what:any[] = []; [where, what] = Object.entries(result[j]); if ((what.length > 1) && (what[1].length > 0)) { const person = where[1] > 0 ? `#${where[1]}` : ''; const label = `${where[0]} ${person}: ${what[1]}`; if (options.shadowColor && options.shadowColor !== '') { ctx.fillStyle = options.shadowColor; ctx.fillText(label, 8, 2 + (i * options.lineHeight)); } ctx.fillStyle = options.labelColor; ctx.fillText(label, 6, 0 + (i * options.lineHeight)); i += 1; } } } export async function face(inCanvas, result) { if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; for (const f of result) { ctx.font = options.font; ctx.strokeStyle = options.color; ctx.fillStyle = options.color; if (options.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); // silly hack since fillText does not suport new line const labels:string[] = []; labels.push(`face confidence: ${Math.trunc(100 * f.confidence)}%`); if (f.genderConfidence) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderConfidence)}% confident`); // if (f.genderConfidence) labels.push(f.gender); if (f.age) labels.push(`age: ${f.age || ''}`); if (f.iris) labels.push(`iris distance: ${f.iris}`); if (f.emotion && f.emotion.length > 0) { const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); labels.push(emotion.join(' ')); } if (labels.length === 0) labels.push('face'); ctx.fillStyle = options.color; for (let i = labels.length - 1; i >= 0; i--) { const x = Math.max(f.box[0], 0); const y = i * options.lineHeight + f.box[1]; if (options.shadowColor && options.shadowColor !== '') { ctx.fillStyle = options.shadowColor; ctx.fillText(labels[i], x + 5, y + 16); } ctx.fillStyle = options.labelColor; ctx.fillText(labels[i], x + 4, y + 15); } ctx.lineWidth = 1; if (f.mesh) { if (options.drawPoints) { for (const pt of f.mesh) { ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color; point(ctx, pt[0], pt[1]); } } if (options.drawPolygons) { for (let i = 0; i < triangulation.length / 3; i++) { const points = [ triangulation[i * 3 + 0], triangulation[i * 3 + 1], triangulation[i * 3 + 2], ].map((index) => f.mesh[index]); ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color; ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)` : options.color; ctx.lineWidth = 1; lines(ctx, points); } // iris: array[center, left, top, right, bottom] if (f.annotations && f.annotations.leftEyeIris) { ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color; ctx.beginPath(); const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2; const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2; ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); if (options.fillPolygons) { ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color; ctx.fill(); } } if (f.annotations && f.annotations.rightEyeIris) { ctx.strokeStyle = options.useDepth ? 'rgba(255, 200, 255, 0.3)' : options.color; ctx.beginPath(); const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2; const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2; ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); if (options.fillPolygons) { ctx.fillStyle = options.useDepth ? 'rgba(255, 255, 200, 0.3)' : options.color; ctx.fill(); } } } } } } const lastDrawnPose:any[] = []; export async function body(inCanvas, result) { if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; ctx.lineJoin = 'round'; for (let i = 0; i < result.length; i++) { // result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5); if (!lastDrawnPose[i] && options.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; ctx.strokeStyle = options.color; ctx.lineWidth = options.lineWidth; if (options.drawPoints) { for (let pt = 0; pt < result[i].keypoints.length; pt++) { ctx.fillStyle = options.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : options.color; if (options.bufferedOutput) { lastDrawnPose[i].keypoints[pt][0] = (lastDrawnPose[i].keypoints[pt][0] + result[i].keypoints[pt].position.x) / 2; lastDrawnPose[i].keypoints[pt][1] = (lastDrawnPose[i].keypoints[pt][1] + result[i].keypoints[pt].position.y) / 2; point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1]); } else { point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y); } } } if (options.drawLabels) { ctx.font = options.font; for (const pt of result[i].keypoints) { ctx.fillStyle = options.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : options.color; ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4); } } if (options.drawPolygons) { const path = new Path2D(); let root; let part; // torso root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); if (root && root.score > config.body.scoreThreshold) { const points: any[] = []; points.push([root.position.x, root.position.y, 'leftShoulder']); part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHip'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHip'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); lines(ctx, points); } // leg left root = result[i].keypoints.find((a) => a.part === 'leftHip'); if (root && root.score > config.body.scoreThreshold) { const points: any[] = []; points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftKnee'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftAnkle'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftHeel'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftFoot'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); lines(ctx, points); } // leg right root = result[i].keypoints.find((a) => a.part === 'rightHip'); if (root && root.score > config.body.scoreThreshold) { const points: any[] = []; points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightKnee'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightAnkle'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightHeel'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightFoot'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); lines(ctx, points); } // arm left root = result[i].keypoints.find((a) => a.part === 'leftShoulder'); if (root && root.score > config.body.scoreThreshold) { const points: any[] = []; points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftElbow'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftWrist'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftPalm'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); lines(ctx, points); } // arm right root = result[i].keypoints.find((a) => a.part === 'rightShoulder'); if (root && root.score > config.body.scoreThreshold) { const points: any[] = []; points.push([root.position.x, root.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightElbow'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightWrist'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightPalm'); if (part && part.score > config.body.scoreThreshold) points.push([part.position.x, part.position.y]); lines(ctx, points); } // draw all ctx.stroke(path); } } } export async function hand(inCanvas, result) { if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; ctx.lineJoin = 'round'; ctx.font = options.font; for (const h of result) { if (options.drawBoxes) { ctx.strokeStyle = options.color; ctx.fillStyle = options.color; rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); if (options.shadowColor && options.shadowColor !== '') { ctx.fillStyle = options.shadowColor; ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + options.lineHeight, h.box[2]); } ctx.fillStyle = options.labelColor; ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + options.lineHeight, h.box[2]); ctx.stroke(); } if (options.drawPoints) { if (h.landmarks && h.landmarks.length > 0) { for (const pt of h.landmarks) { ctx.fillStyle = options.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : options.color; point(ctx, pt[0], pt[1]); } } } if (options.drawPolygons) { const addPart = (part) => { if (!part) return; for (let i = 0; i < part.length; i++) { ctx.lineWidth = options.lineWidth; ctx.beginPath(); ctx.strokeStyle = options.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : options.color; ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]); ctx.lineTo(part[i][0], part[i][1]); ctx.stroke(); } }; addPart(h.annotations.indexFinger); addPart(h.annotations.middleFinger); addPart(h.annotations.ringFinger); addPart(h.annotations.pinky); addPart(h.annotations.thumb); // addPart(hand.annotations.palmBase); } } } export async function canvas(inCanvas, outCanvas) { if (!inCanvas || !outCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement) || !(outCanvas instanceof HTMLCanvasElement)) return; const outCtx = inCanvas.getContext('2d'); outCtx?.drawImage(inCanvas, 0, 0); } export async function all(inCanvas, result) { if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; face(inCanvas, result.face); body(inCanvas, result.body); hand(inCanvas, result.hand); gesture(inCanvas, result.gesture); }