add draw label templates

pull/356/head
Vladimir Mandic 2022-10-18 10:18:40 -04:00
parent 510e89d9f2
commit afb70c52e0
17 changed files with 202 additions and 159 deletions

View File

@ -9,8 +9,9 @@
## Changelog
### **HEAD -> main** 2022/10/16 mandic00@live.com
### **HEAD -> main** 2022/10/17 mandic00@live.com
- tensor rank strong typechecks
### **origin/main** 2022/10/13 mandic00@live.com

View File

@ -108,6 +108,7 @@
- [**Usage & Functions**](https://github.com/vladmandic/human/wiki/Usage)
- [**Configuration Details**](https://github.com/vladmandic/human/wiki/Config)
- [**Result Details**](https://github.com/vladmandic/human/wiki/Result)
- [**Customizing Draw Methods**](https://github.com/vladmandic/human/wiki/Draw)
- [**Caching & Smoothing**](https://github.com/vladmandic/human/wiki/Caching)
- [**Input Processing**](https://github.com/vladmandic/human/wiki/Image)
- [**Face Recognition & Face Description**](https://github.com/vladmandic/human/wiki/Embedding)

View File

@ -2,8 +2,6 @@
## Work-in-Progress
- `src/tfjs/tfjs.esm.d.ts`
- `src/tfjs/long.d.ts`
<hr><br>

View File

@ -4,6 +4,6 @@
author: <https://github.com/vladmandic>'
*/
import*as i from"../../dist/human.esm.js";var m={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1,flip:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},segmentation:{enabled:!1},gesture:{enabled:!0}},e=new i.Human(m);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;var a={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},s={detectFPS:0,drawFPS:0,frames:0,averageMs:0},o=(...t)=>{a.log.innerText+=t.join(" ")+`
`,console.log(...t)},d=t=>a.fps.innerText=t,f=t=>a.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(t).replace(/"|{|}/g,"").replace(/,/g," | ");async function l(){if(!a.video.paused){n.start===0&&(n.start=e.now()),await e.detect(a.video);let t=e.tf.memory().numTensors;t-n.tensors!==0&&o("allocated tensors:",t-n.tensors),n.tensors=t,s.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,s.frames++,s.averageMs=Math.round(1e3*(e.now()-n.start)/s.frames)/1e3,s.frames%100===0&&!a.video.paused&&o("performance",{...s,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(l)}async function c(){if(!a.video.paused){let r=e.next(e.result);e.config.filter.flip?e.draw.canvas(r.canvas,a.canvas):e.draw.canvas(a.video,a.canvas),await e.draw.all(a.canvas,r),f(r.performance)}let t=e.now();s.drawFPS=Math.round(1e3*1e3/(t-n.draw))/1e3,n.draw=t,d(a.video.paused?"paused":`fps: ${s.detectFPS.toFixed(1).padStart(5," ")} detect | ${s.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(c,30)}async function u(){await e.webcam.start({element:a.video,crop:!0}),a.canvas.width=e.webcam.width,a.canvas.height=e.webcam.height,a.canvas.onclick=async()=>{e.webcam.paused?await e.webcam.play():e.webcam.pause()}}async function w(){o("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),o("platform:",e.env.platform,"| agent:",e.env.agent),d("loading..."),await e.load(),o("backend:",e.tf.getBackend(),"| available:",e.env.backends),o("models stats:",e.getModelStats()),o("models loaded:",Object.values(e.models).filter(t=>t!==null).length),d("initializing..."),await e.warmup(),await u(),await l(),await c()}window.onload=w;
import*as m from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1,flip:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0},antispoof:{enabled:!0},liveness:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},segmentation:{enabled:!1},gesture:{enabled:!0}},e=new m.Human(w);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;var a={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},s={detectFPS:0,drawFPS:0,frames:0,averageMs:0},o=(...t)=>{a.log.innerText+=t.join(" ")+`
`,console.log(...t)},d=t=>a.fps.innerText=t,v=t=>a.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(t).replace(/"|{|}/g,"").replace(/,/g," | ");async function f(){if(!a.video.paused){n.start===0&&(n.start=e.now()),await e.detect(a.video);let t=e.tf.memory().numTensors;t-n.tensors!==0&&o("allocated tensors:",t-n.tensors),n.tensors=t,s.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,s.frames++,s.averageMs=Math.round(1e3*(e.now()-n.start)/s.frames)/1e3,s.frames%100===0&&!a.video.paused&&o("performance",{...s,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(f)}async function u(){var i,l,c;if(!a.video.paused){let r=e.next(e.result);e.config.filter.flip?e.draw.canvas(r.canvas,a.canvas):e.draw.canvas(a.video,a.canvas);let p={bodyLabels:`person confidence [score] and ${(c=(l=(i=e.result)==null?void 0:i.body)==null?void 0:l[0])==null?void 0:c.keypoints.length} keypoints`};await e.draw.all(a.canvas,r,p),v(r.performance)}let t=e.now();s.drawFPS=Math.round(1e3*1e3/(t-n.draw))/1e3,n.draw=t,d(a.video.paused?"paused":`fps: ${s.detectFPS.toFixed(1).padStart(5," ")} detect | ${s.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(u,30)}async function g(){await e.webcam.start({element:a.video,crop:!0}),a.canvas.width=e.webcam.width,a.canvas.height=e.webcam.height,a.canvas.onclick=async()=>{e.webcam.paused?await e.webcam.play():e.webcam.pause()}}async function b(){o("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),o("platform:",e.env.platform,"| agent:",e.env.agent),d("loading..."),await e.load(),o("backend:",e.tf.getBackend(),"| available:",e.env.backends),o("models stats:",e.getModelStats()),o("models loaded:",Object.values(e.models).filter(t=>t!==null).length),o("environment",e.env),d("initializing..."),await e.warmup(),await g(),await f(),await u()}window.onload=b;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@ -10,9 +10,10 @@
import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human
const humanConfig: Partial<H.Config> = { // user configuration for human, used to fine-tune behavior
// backend: 'wasm',
modelBasePath: '../../models',
filter: { enabled: true, equalization: false, flip: false },
face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } },
body: { enabled: true },
hand: { enabled: true },
object: { enabled: false },
@ -66,7 +67,9 @@ async function drawLoop() { // main screen refresh loop
const interpolated = human.next(human.result); // smoothen result using last-known results
if (human.config.filter.flip) human.draw.canvas(interpolated.canvas as HTMLCanvasElement, dom.canvas); // draw processed image to screen canvas
else human.draw.canvas(dom.video, dom.canvas); // draw original video to screen canvas // better than using procesed image as this loop happens faster than processing loop
await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.
const opt: Partial<H.DrawOptions> = { bodyLabels: `person confidence [score] and ${human.result?.body?.[0]?.keypoints.length} keypoints` };
await human.draw.all(dom.canvas, interpolated, opt); // draw labels, boxes, lines, etc.
perf(interpolated.performance); // write performance data
}
const now = human.now();
@ -94,6 +97,7 @@ async function main() { // main entry point
log('backend:', human.tf.getBackend(), '| available:', human.env.backends);
log('models stats:', human.getModelStats());
log('models loaded:', Object.values(human.models).filter((model) => model !== null).length);
log('environment', human.env);
status('initializing...');
await human.warmup(); // warmup function to initialize backend for future faster detection
await webCam(); // start webcam

View File

@ -1,5 +1,5 @@
import { mergeDeep } from '../util/util';
import { getCanvasContext, rect, point, curves, colorDepth } from './primitives';
import { getCanvasContext, rect, point, curves, colorDepth, replace, labels } from './primitives';
import { options } from './options';
import type { BodyResult } from '../result';
import type { AnyCanvas, DrawOptions } from '../exports';
@ -18,13 +18,10 @@ export function body(inCanvas: AnyCanvas, result: BodyResult[], drawOptions?: Pa
ctx.font = localOptions.font;
if (localOptions.drawBoxes && result[i].box && result[i].box.length === 4) {
rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions);
if (localOptions.drawLabels) {
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]);
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]);
if (localOptions.drawLabels && (localOptions.bodyLabels?.length > 0)) {
let l = localOptions.bodyLabels.slice();
l = replace(l, '[score]', 100 * result[i].score);
labels(ctx, l, result[i].box[0], result[i].box[1], localOptions);
}
}
if (localOptions.drawPoints && result[i].keypoints) {
@ -34,12 +31,14 @@ export function body(inCanvas: AnyCanvas, result: BodyResult[], drawOptions?: Pa
point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions);
}
}
if (localOptions.drawLabels && result[i].keypoints) {
if (localOptions.drawLabels && (localOptions.bodyPartLabels?.length > 0) && result[i].keypoints) {
ctx.font = localOptions.font;
for (const pt of result[i].keypoints) {
if (!pt.score || (pt.score === 0)) continue;
ctx.fillStyle = colorDepth(pt.position[2], localOptions);
ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4);
let l = localOptions.bodyPartLabels.slice();
l = replace(l, '[label]', pt.part);
l = replace(l, '[score]', 100 * pt.score);
labels(ctx, l, pt.position[0], pt.position[1], localOptions);
}
}
if (localOptions.drawPolygons && result[i].keypoints && result[i].annotations) {

View File

@ -11,6 +11,7 @@ import { body } from './body';
import { hand } from './hand';
import { object } from './object';
import { gesture } from './gesture';
import { defaultLabels } from './labels';
import type { Result, PersonResult } from '../result';
import type { AnyCanvas, DrawOptions } from '../exports';
@ -76,3 +77,14 @@ export async function all(inCanvas: AnyCanvas, result: Result, drawOptions?: Par
result.performance.draw = drawTime;
return promise;
}
/** sets default label templates for face/body/hand/object/gestures */
export function init() {
options.faceLabels = defaultLabels.face;
options.bodyLabels = defaultLabels.body;
options.bodyPartLabels = defaultLabels.bodyPart;
options.handLabels = defaultLabels.hand;
options.fingerLabels = defaultLabels.finger;
options.objectLabels = defaultLabels.object;
options.gestureLabels = defaultLabels.gesture;
}

View File

@ -1,77 +1,65 @@
import { TRI468 as triangulation } from '../face/facemeshcoords';
import { mergeDeep } from '../util/util';
import { getCanvasContext, rad2deg, rect, point, lines, arrow } from './primitives';
import { getCanvasContext, rad2deg, rect, point, lines, arrow, labels, replace } from './primitives';
import { options } from './options';
import * as facemeshConstants from '../face/constants';
import type { FaceResult } from '../result';
import type { AnyCanvas, DrawOptions } from '../exports';
let opt: DrawOptions;
let localOptions: DrawOptions;
function drawLabels(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
if (opt.drawLabels) {
// silly hack since fillText does not suport new line
const labels:string[] = [];
labels.push(`face: ${Math.trunc(100 * f.score)}%`);
if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}%`);
if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`distance: ${f.iris}`);
if (f.real) labels.push(`real: ${Math.trunc(100 * f.real)}%`);
if (f.live) labels.push(`live: ${Math.trunc(100 * f.live)}%`);
if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
if (emotion.length > 3) emotion.length = 3;
labels.push(emotion.join(' '));
}
if (f.rotation?.angle && f.rotation?.gaze) {
if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`);
if (f.rotation.gaze.bearing) labels.push(`gaze: ${rad2deg(f.rotation.gaze.bearing)}°`);
}
if (labels.length === 0) labels.push('face');
ctx.fillStyle = opt.color;
for (let i = labels.length - 1; i >= 0; i--) {
const x = Math.max(f.box[0], 0);
const y = i * opt.lineHeight + f.box[1];
if (opt.shadowColor && opt.shadowColor !== '') {
ctx.fillStyle = opt.shadowColor;
ctx.fillText(labels[i], x + 5, y + 16);
}
ctx.fillStyle = opt.labelColor;
ctx.fillText(labels[i], x + 4, y + 15);
}
if (!localOptions.drawLabels || (localOptions.faceLabels?.length === 0)) return;
let l = localOptions.faceLabels.slice();
if (f.score) l = replace(l, '[score]', 100 * f.score);
if (f.gender) l = replace(l, '[gender]', f.gender);
if (f.genderScore) l = replace(l, '[genderScore]', 100 * f.genderScore);
if (f.age) l = replace(l, '[age]', f.age);
if (f.iris) l = replace(l, '[distance]', f.iris);
if (f.real) l = replace(l, '[real]', 100 * f.real);
if (f.live) l = replace(l, '[live]', 100 * f.live);
if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
if (emotion.length > 3) emotion.length = 3;
l = replace(l, '[emotions]', emotion.join(' '));
}
if (f.rotation?.angle?.roll) l = replace(l, '[roll]', rad2deg(f.rotation.angle.roll));
if (f.rotation?.angle?.yaw) l = replace(l, '[yaw]', rad2deg(f.rotation.angle.yaw));
if (f.rotation?.angle?.pitch) l = replace(l, '[pitch]', rad2deg(f.rotation.angle.pitch));
if (f.rotation?.gaze?.bearing) l = replace(l, '[gaze]', rad2deg(f.rotation.gaze.bearing));
labels(ctx, l, f.box[0], f.box[1], localOptions);
}
function drawIrisElipse(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
// iris: array[center, left, top, right, bottom]
if (f.annotations?.leftEyeIris && f.annotations?.leftEyeIris[0]) {
ctx.strokeStyle = opt.useDepth ? 'rgba(255, 200, 255, 0.3)' : opt.color;
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
ctx.beginPath();
const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2;
const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2;
ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (opt.fillPolygons) {
ctx.fillStyle = opt.useDepth ? 'rgba(255, 255, 200, 0.3)' : opt.color;
if (localOptions.fillPolygons) {
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
ctx.fill();
}
}
if (f.annotations?.rightEyeIris && f.annotations?.rightEyeIris[0]) {
ctx.strokeStyle = opt.useDepth ? 'rgba(255, 200, 255, 0.3)' : opt.color;
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
ctx.beginPath();
const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2;
const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2;
ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (opt.fillPolygons) {
ctx.fillStyle = opt.useDepth ? 'rgba(255, 255, 200, 0.3)' : opt.color;
if (localOptions.fillPolygons) {
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
ctx.fill();
}
}
}
function drawGazeSpheres(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
if (opt.drawGaze && f.rotation?.angle && typeof Path2D !== 'undefined') {
if (localOptions.drawGaze && f.rotation?.angle && typeof Path2D !== 'undefined') {
ctx.strokeStyle = 'pink';
const valX = (f.box[0] + f.box[2] / 2) - (f.box[3] * rad2deg(f.rotation.angle.yaw) / 90);
const valY = (f.box[1] + f.box[3] / 2) + (f.box[2] * rad2deg(f.rotation.angle.pitch) / 90);
@ -95,7 +83,7 @@ function drawGazeSpheres(f: FaceResult, ctx: CanvasRenderingContext2D | Offscree
}
function drawGazeArrows(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
if (opt.drawGaze && f.rotation?.gaze.strength && f.rotation.gaze.bearing && f.annotations.leftEyeIris && f.annotations.rightEyeIris && f.annotations.leftEyeIris[0] && f.annotations.rightEyeIris[0]) {
if (localOptions.drawGaze && f.rotation?.gaze.strength && f.rotation.gaze.bearing && f.annotations.leftEyeIris && f.annotations.rightEyeIris && f.annotations.leftEyeIris[0] && f.annotations.rightEyeIris[0]) {
ctx.strokeStyle = 'pink';
ctx.fillStyle = 'pink';
const leftGaze = [
@ -112,16 +100,16 @@ function drawGazeArrows(f: FaceResult, ctx: CanvasRenderingContext2D | Offscreen
}
function drawFacePolygons(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
if (opt.drawPolygons && f.mesh.length >= 468) {
if (localOptions.drawPolygons && f.mesh.length >= 468) {
ctx.lineWidth = 1;
for (let i = 0; i < triangulation.length / 3; i++) {
const points = [triangulation[i * 3 + 0], triangulation[i * 3 + 1], triangulation[i * 3 + 2]].map((index) => f.mesh[index]);
lines(ctx, points, opt);
lines(ctx, points, localOptions);
}
drawIrisElipse(f, ctx);
}
/*
if (opt.drawPolygons && f.contours.length > 1) {
if (localOptions.drawPolygons && f.contours.length > 1) {
ctx.lineWidth = 5;
lines(ctx, f.contours, opt);
}
@ -130,33 +118,33 @@ function drawFacePolygons(f: FaceResult, ctx: CanvasRenderingContext2D | Offscre
}
function drawFacePoints(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {
if (opt.drawPoints && f.mesh.length >= 468) {
if (localOptions.drawPoints && f.mesh.length >= 468) {
for (let i = 0; i < f.mesh.length; i++) {
point(ctx, f.mesh[i][0], f.mesh[i][1], f.mesh[i][2], opt);
if (opt.drawAttention) {
if (facemeshConstants.LANDMARKS_REFINEMENT_LIPS_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) + 127, opt);
if (facemeshConstants.LANDMARKS_REFINEMENT_LEFT_EYE_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) - 127, opt);
if (facemeshConstants.LANDMARKS_REFINEMENT_RIGHT_EYE_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) - 127, opt);
point(ctx, f.mesh[i][0], f.mesh[i][1], f.mesh[i][2], localOptions);
if (localOptions.drawAttention) {
if (facemeshConstants.LANDMARKS_REFINEMENT_LIPS_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) + 127, localOptions);
if (facemeshConstants.LANDMARKS_REFINEMENT_LEFT_EYE_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) - 127, localOptions);
if (facemeshConstants.LANDMARKS_REFINEMENT_RIGHT_EYE_CONFIG.includes(i)) point(ctx, f.mesh[i][0], f.mesh[i][1], (f.mesh[i][2] as number) - 127, localOptions);
}
}
}
}
function drawFaceBoxes(f: FaceResult, ctx) {
if (opt.drawBoxes) {
rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], opt);
if (localOptions.drawBoxes) {
rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions);
}
}
/** draw detected faces */
export function face(inCanvas: AnyCanvas, result: FaceResult[], drawOptions?: Partial<DrawOptions>) {
opt = mergeDeep(options, drawOptions);
localOptions = mergeDeep(options, drawOptions);
if (!result || !inCanvas) return;
const ctx = getCanvasContext(inCanvas);
if (!ctx) return;
ctx.font = opt.font;
ctx.strokeStyle = opt.color;
ctx.fillStyle = opt.color;
ctx.font = localOptions.font;
ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color;
for (const f of result) {
drawFaceBoxes(f, ctx);
drawLabels(f, ctx);

View File

@ -1,5 +1,5 @@
import { mergeDeep } from '../util/util';
import { getCanvasContext } from './primitives';
import { getCanvasContext, replace, labels } from './primitives';
import { options } from './options';
import type { GestureResult } from '../result';
import type { AnyCanvas, DrawOptions } from '../exports';
@ -8,25 +8,21 @@ import type { AnyCanvas, DrawOptions } from '../exports';
export function gesture(inCanvas: AnyCanvas, result: GestureResult[], drawOptions?: Partial<DrawOptions>) {
const localOptions: DrawOptions = mergeDeep(options, drawOptions);
if (!result || !inCanvas) return;
if (localOptions.drawGestures) {
if (localOptions.drawGestures && (localOptions.gestureLabels?.length > 0)) {
const ctx = getCanvasContext(inCanvas);
if (!ctx) return;
ctx.font = localOptions.font;
ctx.fillStyle = localOptions.color;
let i = 1;
for (let j = 0; j < result.length; j++) {
let where: unknown[] = []; // what&where is a record
let what: unknown[] = []; // what&where is a record
[where, what] = Object.entries(result[j]);
const [where, what] = Object.entries(result[j]);
if ((what.length > 1) && ((what[1] as string).length > 0)) {
const who = where[1] as number > 0 ? `#${where[1]}` : '';
const label = `${where[0]} ${who}: ${what[1]}`;
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText(label, 8, 2 + (i * localOptions.lineHeight));
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText(label, 6, 0 + (i * localOptions.lineHeight));
let l = localOptions.gestureLabels.slice();
l = replace(l, '[where]', where[0]);
l = replace(l, '[who]', who);
l = replace(l, '[what]', what[1]);
labels(ctx, l, 8, 2 + (i * localOptions.lineHeight), localOptions);
i += 1;
}
}

View File

@ -1,5 +1,5 @@
import { mergeDeep } from '../util/util';
import { getCanvasContext, rect, point, colorDepth } from './primitives';
import { getCanvasContext, rect, point, colorDepth, replace, labels } from './primitives';
import { options } from './options';
import type { HandResult } from '../result';
import type { AnyCanvas, DrawOptions, Point } from '../exports';
@ -17,13 +17,11 @@ export function hand(inCanvas: AnyCanvas, result: HandResult[], drawOptions?: Pa
ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color;
rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
if (localOptions.drawLabels) {
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText(`hand:${Math.trunc(100 * h.score)}%`, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); // can use h.label
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText(`hand:${Math.trunc(100 * h.score)}%`, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); // can use h.label
if (localOptions.drawLabels && (localOptions.handLabels?.length > 0)) {
let l = localOptions.handLabels.slice();
l = replace(l, '[label]', h.label);
l = replace(l, '[score]', 100 * h.score);
labels(ctx, l, h.box[0], h.box[1], localOptions);
}
ctx.stroke();
}
@ -35,20 +33,12 @@ export function hand(inCanvas: AnyCanvas, result: HandResult[], drawOptions?: Pa
}
}
}
if (localOptions.drawLabels && h.annotations) {
const addHandLabel = (part: Point[], title: string) => {
if (!part || part.length === 0 || !part[0]) return;
const z = part[part.length - 1][2] || -256;
ctx.fillStyle = colorDepth(z, localOptions);
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
};
ctx.font = localOptions.font;
addHandLabel(h.annotations.index, 'index');
addHandLabel(h.annotations.middle, 'middle');
addHandLabel(h.annotations.ring, 'ring');
addHandLabel(h.annotations.pinky, 'pinky');
addHandLabel(h.annotations.thumb, 'thumb');
addHandLabel(h.annotations.palm, 'palm');
if (localOptions.drawLabels && h.annotations && (localOptions.fingerLabels?.length > 0)) {
for (const [part, pt] of Object.entries(h.annotations)) {
let l = localOptions.fingerLabels.slice();
l = replace(l, '[label]', part);
labels(ctx, l, pt[pt.length - 1][0], pt[pt.length - 1][1], localOptions);
}
}
if (localOptions.drawPolygons && h.annotations) {
const addHandLine = (part: Point[]) => {

18
src/draw/labels.ts Normal file
View File

@ -0,0 +1,18 @@
export const defaultLabels = {
face: `face
confidence: [score]%
[gender] [genderScore]%
age: [age] years
distance: [distance]cm
real: [real]%
live: [live]%
[emotions]
roll: [roll]° yaw:[yaw]° pitch:[pitch]°
gaze: [gaze]°`,
body: 'body [score]%',
bodyPart: '[label] [score]%',
object: '[label] [score]%',
hand: '[label] [score]%',
finger: '[label]',
gesture: '[where] [who]: [what]',
};

View File

@ -1,5 +1,5 @@
import { mergeDeep } from '../util/util';
import { getCanvasContext, rect } from './primitives';
import { getCanvasContext, rect, replace, labels } from './primitives';
import { options } from './options';
import type { ObjectResult } from '../result';
import type { AnyCanvas, DrawOptions } from '../exports';
@ -17,14 +17,11 @@ export function object(inCanvas: AnyCanvas, result: ObjectResult[], drawOptions?
ctx.strokeStyle = localOptions.color;
ctx.fillStyle = localOptions.color;
rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
if (localOptions.drawLabels) {
const label = `${h.label} ${Math.round(100 * h.score)}%`;
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]);
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]);
if (localOptions.drawLabels && (localOptions.objectLabels?.length > 0)) {
let l = localOptions.objectLabels.slice();
l = replace(l, '[label]', h.label);
l = replace(l, '[score]', 100 * h.score);
labels(ctx, l, h.box[0], h.box[1], localOptions);
}
ctx.stroke();
}

View File

@ -1,6 +1,7 @@
/** Draw Options
* - Accessed via `human.draw.options` or provided per each draw method as the drawOptions optional parameter
*/
export interface DrawOptions {
/** draw line color */
color: string,
@ -40,6 +41,20 @@ export interface DrawOptions {
useDepth: boolean,
/** should lines be curved? */
useCurves: boolean,
/** string template for face labels */
faceLabels: string,
/** string template for body labels */
bodyLabels: string,
/** string template for body part labels */
bodyPartLabels: string,
/** string template for hand labels */
handLabels: string,
/** string template for hand labels */
fingerLabels: string,
/** string template for object labels */
objectLabels: string,
/** string template for gesture labels */
gestureLabels: string,
}
/** currently set draw options {@link DrawOptions} */
@ -63,4 +78,11 @@ export const options: DrawOptions = {
fillPolygons: false as boolean,
useDepth: true as boolean,
useCurves: false as boolean,
faceLabels: '' as string,
bodyLabels: '' as string,
bodyPartLabels: '' as string,
objectLabels: '' as string,
handLabels: '' as string,
fingerLabels: '' as string,
gestureLabels: '' as string,
};

View File

@ -16,12 +16,28 @@ export const getCanvasContext = (input: AnyCanvas) => {
export const rad2deg = (theta: number) => Math.round((theta * 180) / Math.PI);
export const replace = (str: string, source: string, target: string | number) => str.replace(source, typeof target === 'number' ? target.toFixed(1) : target);
export const colorDepth = (z: number | undefined, opt: DrawOptions): string => { // performance optimization needed
if (!opt.useDepth || typeof z === 'undefined') return opt.color;
const rgb = Uint8ClampedArray.from([127 + (2 * z), 127 - (2 * z), 255]);
return `rgba(${rgb[0]}, ${rgb[1]}, ${rgb[2]}, ${opt.alpha})`;
};
export function labels(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, str: string, startX: number, startY: number, localOptions: DrawOptions) {
const line: string[] = str.replace(/\[.*\]/g, '').split('\n').map((l) => l.trim()); // remove unmatched templates and split into array
const x = Math.max(0, startX);
for (let i = line.length - 1; i >= 0; i--) {
const y = i * localOptions.lineHeight + startY;
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
ctx.fillStyle = localOptions.shadowColor;
ctx.fillText(line[i], x + 5, y + 16);
}
ctx.fillStyle = localOptions.labelColor;
ctx.fillText(line[i], x + 4, y + 15);
}
}
export function point(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, x: number, y: number, z: number | undefined, localOptions: DrawOptions) {
ctx.fillStyle = colorDepth(z, localOptions);
ctx.beginPath();

View File

@ -157,6 +157,7 @@ export class Human {
// object that contains all initialized models
this.models = new models.Models();
// reexport draw methods
draw.init();
this.draw = {
options: draw.options,
canvas: (input: AnyCanvas | HTMLImageElement | HTMLVideoElement, output: AnyCanvas) => draw.canvas(input, output),

View File

@ -1,40 +1,40 @@
2022-10-17 10:45:30 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"}
2022-10-17 10:45:30 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"}
2022-10-17 10:45:30 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-10-17 10:45:30 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.11","typescript":"4.8.4","typedoc":"0.23.16","eslint":"8.25.0"}
2022-10-17 10:45:30 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-10-17 10:45:30 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":78,"inputBytes":670926,"outputBytes":315728}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":78,"inputBytes":670930,"outputBytes":315732}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":78,"inputBytes":671878,"outputBytes":315843}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":78,"inputBytes":670672,"outputBytes":314337}
2022-10-17 10:45:30 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144854}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":78,"inputBytes":1814856,"outputBytes":1455805}
2022-10-17 10:45:30 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":78,"inputBytes":1814856,"outputBytes":1912566}
2022-10-17 10:45:34 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15}
2022-10-17 10:45:36 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true}
2022-10-17 10:45:36 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5672,"outputBytes":2632}
2022-10-17 10:45:36 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17134,"outputBytes":9181}
2022-10-17 10:45:44 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":115,"errors":0,"warnings":0}
2022-10-17 10:45:44 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-10-17 10:45:44 STATE: Copy: {"input":"src/tfjs","output":"dist/tfjs.esm.d.ts"}
2022-10-17 10:45:44 INFO:  Done...
2022-10-17 10:45:45 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195}
2022-10-17 10:45:45 STATE: Filter: {"input":"types/human.d.ts"}
2022-10-17 10:45:45 STATE: Link: {"input":"types/human.d.ts"}
2022-10-17 10:45:45 INFO:  Analyze models: {"folders":8,"result":"models/models.json"}
2022-10-17 10:45:45 STATE: Models {"folder":"./models","models":12}
2022-10-17 10:45:45 STATE: Models {"folder":"../human-models/models","models":43}
2022-10-17 10:45:45 STATE: Models {"folder":"../blazepose/model/","models":4}
2022-10-17 10:45:45 STATE: Models {"folder":"../anti-spoofing/model","models":1}
2022-10-17 10:45:45 STATE: Models {"folder":"../efficientpose/models","models":3}
2022-10-17 10:45:45 STATE: Models {"folder":"../insightface/models","models":5}
2022-10-17 10:45:45 STATE: Models {"folder":"../movenet/models","models":3}
2022-10-17 10:45:45 STATE: Models {"folder":"../nanodet/models","models":4}
2022-10-17 10:45:45 STATE: Models: {"count":58,"totalSize":386543911}
2022-10-17 10:45:45 INFO:  Human Build complete... {"logFile":"test/build.log"}
2022-10-18 10:18:08 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"}
2022-10-18 10:18:08 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"}
2022-10-18 10:18:08 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-10-18 10:18:08 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.11","typescript":"4.8.4","typedoc":"0.23.16","eslint":"8.25.0"}
2022-10-18 10:18:08 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-10-18 10:18:08 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":79,"inputBytes":672207,"outputBytes":316305}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":79,"inputBytes":672211,"outputBytes":316309}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":79,"inputBytes":673159,"outputBytes":316420}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":79,"inputBytes":671953,"outputBytes":314910}
2022-10-18 10:18:09 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144854}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":79,"inputBytes":1816137,"outputBytes":1456384}
2022-10-18 10:18:09 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":79,"inputBytes":1816137,"outputBytes":1913669}
2022-10-18 10:18:13 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15}
2022-10-18 10:18:15 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true}
2022-10-18 10:18:15 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5936,"outputBytes":2867}
2022-10-18 10:18:15 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17134,"outputBytes":9181}
2022-10-18 10:18:24 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":116,"errors":0,"warnings":0}
2022-10-18 10:18:24 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-10-18 10:18:24 STATE: Copy: {"input":"src/tfjs","output":"dist/tfjs.esm.d.ts"}
2022-10-18 10:18:24 INFO:  Done...
2022-10-18 10:18:24 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195}
2022-10-18 10:18:24 STATE: Filter: {"input":"types/human.d.ts"}
2022-10-18 10:18:24 STATE: Link: {"input":"types/human.d.ts"}
2022-10-18 10:18:24 INFO:  Analyze models: {"folders":8,"result":"models/models.json"}
2022-10-18 10:18:24 STATE: Models {"folder":"./models","models":12}
2022-10-18 10:18:24 STATE: Models {"folder":"../human-models/models","models":43}
2022-10-18 10:18:24 STATE: Models {"folder":"../blazepose/model/","models":4}
2022-10-18 10:18:24 STATE: Models {"folder":"../anti-spoofing/model","models":1}
2022-10-18 10:18:24 STATE: Models {"folder":"../efficientpose/models","models":3}
2022-10-18 10:18:24 STATE: Models {"folder":"../insightface/models","models":5}
2022-10-18 10:18:24 STATE: Models {"folder":"../movenet/models","models":3}
2022-10-18 10:18:24 STATE: Models {"folder":"../nanodet/models","models":4}
2022-10-18 10:18:25 STATE: Models: {"count":58,"totalSize":386543911}
2022-10-18 10:18:25 INFO:  Human Build complete... {"logFile":"test/build.log"}