mirror of https://github.com/vladmandic/human
refactored human.config and human.draw
Vladimir Mandic
parent
8ab49c7440
commit
0af73ab567
|
|
@ -1,6 +1,6 @@
|
|||
# @vladmandic/human
|
||||
|
||||
Version: **1.4.2**
|
||||
Version: **1.5.0**
|
||||
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
|
||||
|
||||
Author: **Vladimir Mandic <mandic00@live.com>**
|
||||
|
|
@ -9,6 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **1.4.3** 2021/04/12 mandic00@live.com
|
||||
|
||||
- implement webrtc
|
||||
|
||||
### **1.4.2** 2021/04/12 mandic00@live.com
|
||||
|
||||
- added support for multiple instances of human
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) fo
|
|||
- [**Code Repository**](https://github.com/vladmandic/human)
|
||||
- [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human)
|
||||
- [**Issues Tracker**](https://github.com/vladmandic/human/issues)
|
||||
- [**API Specification: Human**](https://vladmandic.github.io/human/typedoc/classes/human.html)
|
||||
- [**API Specification: Root**](https://vladmandic.github.io/human/typedoc/)
|
||||
- [**TypeDoc API Specification: Human**](https://vladmandic.github.io/human/typedoc/classes/human.html)
|
||||
- [**TypeDoc API Specification: Root**](https://vladmandic.github.io/human/typedoc/)
|
||||
- [**Change Log**](https://github.com/vladmandic/human/blob/main/CHANGELOG.md)
|
||||
|
||||
## Wiki pages
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const userConfig = {
|
|||
enabled: true,
|
||||
detector: { rotation: true, return: true },
|
||||
mesh: { enabled: true },
|
||||
embedding: { enabled: true },
|
||||
embedding: { enabled: false },
|
||||
iris: { enabled: false },
|
||||
age: { enabled: false },
|
||||
gender: { enabled: false },
|
||||
|
|
|
|||
|
|
@ -467,13 +467,13 @@ function setupMenu() {
|
|||
setupCamera();
|
||||
});
|
||||
menu.display.addHTML('<hr style="border-style: inset; border-color: dimgray">');
|
||||
menu.display.addBool('use 3D depth', human.draw.drawOptions, 'useDepth');
|
||||
menu.display.addBool('draw with curves', human.draw.drawOptions, 'useCurves');
|
||||
menu.display.addBool('print labels', human.draw.drawOptions, 'drawLabels');
|
||||
menu.display.addBool('draw points', human.draw.drawOptions, 'drawPoints');
|
||||
menu.display.addBool('draw boxes', human.draw.drawOptions, 'drawBoxes');
|
||||
menu.display.addBool('draw polygons', human.draw.drawOptions, 'drawPolygons');
|
||||
menu.display.addBool('fill polygons', human.draw.drawOptions, 'fillPolygons');
|
||||
menu.display.addBool('use 3D depth', human.draw.options, 'useDepth');
|
||||
menu.display.addBool('draw with curves', human.draw.options, 'useCurves');
|
||||
menu.display.addBool('print labels', human.draw.options, 'drawLabels');
|
||||
menu.display.addBool('draw points', human.draw.options, 'drawPoints');
|
||||
menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes');
|
||||
menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons');
|
||||
menu.display.addBool('fill polygons', human.draw.options, 'fillPolygons');
|
||||
|
||||
menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] });
|
||||
menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@vladmandic/human",
|
||||
"version": "1.4.3",
|
||||
"version": "1.5.0",
|
||||
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
|
||||
"sideEffects": false,
|
||||
"main": "dist/human.node.js",
|
||||
|
|
@ -67,13 +67,14 @@
|
|||
"@vladmandic/pilogger": "^0.2.16",
|
||||
"chokidar": "^3.5.1",
|
||||
"dayjs": "^1.10.4",
|
||||
"esbuild": "^0.11.9",
|
||||
"esbuild": "^0.11.10",
|
||||
"eslint": "^7.24.0",
|
||||
"eslint-config-airbnb-base": "^14.2.1",
|
||||
"eslint-plugin-import": "^2.22.1",
|
||||
"eslint-plugin-json": "^2.1.2",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"eslint-plugin-promise": "^5.1.0",
|
||||
"hint": "^6.1.3",
|
||||
"rimraf": "^3.0.2",
|
||||
"seedrandom": "^3.0.5",
|
||||
"simple-git": "^2.37.0",
|
||||
|
|
|
|||
108
src/config.ts
108
src/config.ts
|
|
@ -7,38 +7,99 @@
|
|||
* Contains all configurable parameters
|
||||
*/
|
||||
export interface Config {
|
||||
backend: string,
|
||||
/** Backend used for TFJS operations */
|
||||
backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow',
|
||||
/** Path to *.wasm files if backend is set to `wasm` */
|
||||
wasmPath: string,
|
||||
/** Print debug statements to console */
|
||||
debug: boolean,
|
||||
/** Perform model loading and inference concurrently or sequentially */
|
||||
async: boolean,
|
||||
/** Collect and print profiling data during inference operations */
|
||||
profile: boolean,
|
||||
/** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */
|
||||
deallocate: boolean,
|
||||
/** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */
|
||||
scoped: boolean,
|
||||
/** Perform additional optimizations when input is video,
|
||||
* - must be disabled for images
|
||||
* - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs
|
||||
* - skips boundary detection for every `skipFrames` frames specified for each model
|
||||
* - while maintaining in-box detection since objects don't change definition as fast */
|
||||
videoOptimized: boolean,
|
||||
warmup: string,
|
||||
/** What to use for `human.warmup()`
|
||||
* - warmup pre-initializes all models for faster inference but can take significant time on startup
|
||||
* - only used for `webgl` and `humangl` backends
|
||||
*/
|
||||
warmup: 'none' | 'face' | 'full' | 'body',
|
||||
/** Base model path (typically starting with file://, http:// or https://) for all models
|
||||
* - individual modelPath values are joined to this path
|
||||
*/
|
||||
modelBasePath: string,
|
||||
/** Run input through image filters before inference
|
||||
* - image filters run with near-zero latency as they are executed on the GPU
|
||||
*/
|
||||
filter: {
|
||||
enabled: boolean,
|
||||
/** Resize input width
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
width: number,
|
||||
/** Resize input height
|
||||
* - if both width and height are set to 0, there is no resizing
|
||||
* - if just one is set, second one is scaled automatically
|
||||
* - if both are set, values are used as-is
|
||||
*/
|
||||
height: number,
|
||||
/** Return processed canvas imagedata in result */
|
||||
return: boolean,
|
||||
/** Range: -1 (darken) to 1 (lighten) */
|
||||
brightness: number,
|
||||
/** Range: -1 (reduce contrast) to 1 (increase contrast) */
|
||||
contrast: number,
|
||||
/** Range: 0 (no sharpening) to 1 (maximum sharpening) */
|
||||
sharpness: number,
|
||||
/** Range: 0 (no blur) to N (blur radius in pixels) */
|
||||
blur: number
|
||||
/** Range: -1 (reduce saturation) to 1 (increase saturation) */
|
||||
saturation: number,
|
||||
/** Range: 0 (no change) to 360 (hue rotation in degrees) */
|
||||
hue: number,
|
||||
/** Image negative */
|
||||
negative: boolean,
|
||||
/** Image sepia colors */
|
||||
sepia: boolean,
|
||||
/** Image vintage colors */
|
||||
vintage: boolean,
|
||||
/** Image kodachrome colors */
|
||||
kodachrome: boolean,
|
||||
/** Image technicolor colors */
|
||||
technicolor: boolean,
|
||||
/** Image polaroid camera effect */
|
||||
polaroid: boolean,
|
||||
/** Range: 0 (no pixelate) to N (number of pixels to pixelate) */
|
||||
pixelate: number,
|
||||
},
|
||||
/** Controlls gesture detection */
|
||||
gesture: {
|
||||
enabled: boolean,
|
||||
},
|
||||
/** Controlls and configures all face-specific options:
|
||||
* - face detection, face mesh detection, age, gender, emotion detection and face description
|
||||
* Parameters:
|
||||
* - enabled: true/false
|
||||
* - modelPath: path for individual face model
|
||||
* - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles
|
||||
* - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance
|
||||
* - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true
|
||||
* - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
* - return extracted face as tensor for futher user processing
|
||||
*/
|
||||
face: {
|
||||
enabled: boolean,
|
||||
detector: {
|
||||
|
|
@ -87,6 +148,13 @@ export interface Config {
|
|||
modelPath: string,
|
||||
},
|
||||
},
|
||||
/** Controlls and configures all body detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: paths for both hand detector model and hand skeleton model
|
||||
* - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance
|
||||
* - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression
|
||||
* - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression
|
||||
*/
|
||||
body: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
|
|
@ -94,6 +162,18 @@ export interface Config {
|
|||
scoreThreshold: number,
|
||||
nmsRadius: number,
|
||||
},
|
||||
/** Controlls and configures all hand detection specific options
|
||||
* - enabled: true/false
|
||||
* - modelPath: paths for both hand detector model and hand skeleton model
|
||||
* - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted
|
||||
* - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true
|
||||
* - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle
|
||||
* - minConfidence: threshold for discarding a prediction
|
||||
* - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression
|
||||
* - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression
|
||||
* - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance
|
||||
* - landmarks: detect hand landmarks or just hand boundary box
|
||||
*/
|
||||
hand: {
|
||||
enabled: boolean,
|
||||
rotation: boolean,
|
||||
|
|
@ -111,6 +191,12 @@ export interface Config {
|
|||
modelPath: string,
|
||||
},
|
||||
},
|
||||
/** Controlls and configures all object detection specific options
|
||||
* - minConfidence: minimum score that detection must have to return as valid object
|
||||
* - iouThreshold: ammount of overlap between two detected objects before one object is removed
|
||||
* - maxResults: maximum number of detections to return
|
||||
* - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true
|
||||
*/
|
||||
object: {
|
||||
enabled: boolean,
|
||||
modelPath: string,
|
||||
|
|
@ -134,16 +220,16 @@ const config: Config = {
|
|||
// this disables per-model performance data but
|
||||
// slightly increases performance
|
||||
// cannot be used if profiling is enabled
|
||||
profile: false, // enable tfjs profiling
|
||||
profile: false, // internal: enable tfjs profiling
|
||||
// this has significant performance impact
|
||||
// only enable for debugging purposes
|
||||
// currently only implemented for age,gender,emotion models
|
||||
deallocate: false, // aggresively deallocate gpu memory after each usage
|
||||
// only valid for webgl backend and only during first call
|
||||
deallocate: false, // internal: aggresively deallocate gpu memory after each usage
|
||||
// only valid for webgl and humangl backend and only during first call
|
||||
// cannot be changed unless library is reloaded
|
||||
// this has significant performance impact
|
||||
// only enable on low-memory devices
|
||||
scoped: false, // enable scoped runs
|
||||
scoped: false, // internal: enable scoped runs
|
||||
// some models *may* have memory leaks,
|
||||
// this wrapps everything in a local scope at a cost of performance
|
||||
// typically not needed
|
||||
|
|
@ -155,7 +241,9 @@ const config: Config = {
|
|||
warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full'
|
||||
// warmup pre-initializes all models for faster inference but can take
|
||||
// significant time on startup
|
||||
filter: {
|
||||
// only used for `webgl` and `humangl` backends
|
||||
filter: { // run input through image filters before inference
|
||||
// image filters run with near-zero latency as they are executed on the GPU
|
||||
enabled: true, // enable image pre-processing filters
|
||||
width: 0, // resize input width
|
||||
height: 0, // resize input height
|
||||
|
|
@ -201,7 +289,7 @@ const config: Config = {
|
|||
// box for updated face analysis as the head probably hasn't moved much
|
||||
// in short time (10 * 1/25 = 0.25 sec)
|
||||
skipInitial: false, // if previous detection resulted in no faces detected,
|
||||
// should skipFrames be reset immediately
|
||||
// should skipFrames be reset immediately to force new detection cycle
|
||||
minConfidence: 0.2, // threshold for discarding a prediction
|
||||
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in
|
||||
// non-maximum suppression (0.1 means drop if overlap 10%)
|
||||
|
|
@ -289,8 +377,8 @@ const config: Config = {
|
|||
// e.g., if model is running st 25 FPS, we can re-use existing bounding
|
||||
// box for updated hand skeleton analysis as the hand probably
|
||||
// hasn't moved much in short time (10 * 1/25 = 0.25 sec)
|
||||
skipInitial: false, // if previous detection resulted in no faces detected,
|
||||
// should skipFrames be reset immediately
|
||||
skipInitial: false, // if previous detection resulted in no hands detected,
|
||||
// should skipFrames be reset immediately to force new detection cycle
|
||||
minConfidence: 0.1, // threshold for discarding a prediction
|
||||
iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much
|
||||
// in non-maximum suppression
|
||||
|
|
|
|||
270
src/draw/draw.ts
270
src/draw/draw.ts
|
|
@ -1,7 +1,49 @@
|
|||
import { defaults } from '../config';
|
||||
import { TRI468 as triangulation } from '../blazeface/coords';
|
||||
import { mergeDeep } from '../helpers';
|
||||
|
||||
export const drawOptions = {
|
||||
/**
|
||||
* Draw Options
|
||||
* Accessed via `human.draw.options` or provided per each draw method as the drawOptions optional parameter
|
||||
* -color: draw color
|
||||
* -labelColor: color for labels
|
||||
* -shadowColor: optional shadow color for labels
|
||||
* -font: font for labels
|
||||
* -lineHeight: line height for labels, used for multi-line labels,
|
||||
* -lineWidth: width of any lines,
|
||||
* -pointSize: size of any point,
|
||||
* -roundRect: for boxes, round corners by this many pixels,
|
||||
* -drawPoints: should points be drawn,
|
||||
* -drawLabels: should labels be drawn,
|
||||
* -drawBoxes: should boxes be drawn,
|
||||
* -drawPolygons: should polygons be drawn,
|
||||
* -fillPolygons: should drawn polygons be filled,
|
||||
* -useDepth: use z-axis coordinate as color shade,
|
||||
* -useCurves: draw polygons as cures or as lines,
|
||||
* -bufferedOutput: experimental: allows to call draw methods multiple times for each detection and interpolate results between results thus achieving smoother animations
|
||||
* -useRawBoxes: Boolean: internal: use non-normalized coordinates when performing draw methods,
|
||||
*/
|
||||
export interface DrawOptions {
|
||||
color: string,
|
||||
labelColor: string,
|
||||
shadowColor: string,
|
||||
font: string,
|
||||
lineHeight: number,
|
||||
lineWidth: number,
|
||||
pointSize: number,
|
||||
roundRect: number,
|
||||
drawPoints: Boolean,
|
||||
drawLabels: Boolean,
|
||||
drawBoxes: Boolean,
|
||||
drawPolygons: Boolean,
|
||||
fillPolygons: Boolean,
|
||||
useDepth: Boolean,
|
||||
useCurves: Boolean,
|
||||
bufferedOutput: Boolean,
|
||||
useRawBoxes: Boolean,
|
||||
}
|
||||
|
||||
export const options: DrawOptions = {
|
||||
color: <string>'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel
|
||||
labelColor: <string>'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel
|
||||
shadowColor: <string>'black',
|
||||
|
|
@ -21,55 +63,55 @@ export const drawOptions = {
|
|||
useRawBoxes: <Boolean>false,
|
||||
};
|
||||
|
||||
function point(ctx, x, y, z = null) {
|
||||
ctx.fillStyle = drawOptions.useDepth && z ? `rgba(${127.5 + (2 * (z || 0))}, ${127.5 - (2 * (z || 0))}, 255, 0.3)` : drawOptions.color;
|
||||
function point(ctx, x, y, z = 0, localOptions) {
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, drawOptions.pointSize, 0, 2 * Math.PI);
|
||||
ctx.arc(x, y, localOptions.pointSize, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
function rect(ctx, x, y, width, height) {
|
||||
function rect(ctx, x, y, width, height, localOptions) {
|
||||
ctx.beginPath();
|
||||
if (drawOptions.useCurves) {
|
||||
if (localOptions.useCurves) {
|
||||
const cx = (x + x + width) / 2;
|
||||
const cy = (y + y + height) / 2;
|
||||
ctx.ellipse(cx, cy, width / 2, height / 2, 0, 0, 2 * Math.PI);
|
||||
} else {
|
||||
ctx.lineWidth = drawOptions.lineWidth;
|
||||
ctx.moveTo(x + drawOptions.roundRect, y);
|
||||
ctx.lineTo(x + width - drawOptions.roundRect, y);
|
||||
ctx.quadraticCurveTo(x + width, y, x + width, y + drawOptions.roundRect);
|
||||
ctx.lineTo(x + width, y + height - drawOptions.roundRect);
|
||||
ctx.quadraticCurveTo(x + width, y + height, x + width - drawOptions.roundRect, y + height);
|
||||
ctx.lineTo(x + drawOptions.roundRect, y + height);
|
||||
ctx.quadraticCurveTo(x, y + height, x, y + height - drawOptions.roundRect);
|
||||
ctx.lineTo(x, y + drawOptions.roundRect);
|
||||
ctx.quadraticCurveTo(x, y, x + drawOptions.roundRect, y);
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
ctx.moveTo(x + localOptions.roundRect, y);
|
||||
ctx.lineTo(x + width - localOptions.roundRect, y);
|
||||
ctx.quadraticCurveTo(x + width, y, x + width, y + localOptions.roundRect);
|
||||
ctx.lineTo(x + width, y + height - localOptions.roundRect);
|
||||
ctx.quadraticCurveTo(x + width, y + height, x + width - localOptions.roundRect, y + height);
|
||||
ctx.lineTo(x + localOptions.roundRect, y + height);
|
||||
ctx.quadraticCurveTo(x, y + height, x, y + height - localOptions.roundRect);
|
||||
ctx.lineTo(x, y + localOptions.roundRect);
|
||||
ctx.quadraticCurveTo(x, y, x + localOptions.roundRect, y);
|
||||
ctx.closePath();
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
function lines(ctx, points: number[] = []) {
|
||||
function lines(ctx, points: number[] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
ctx.strokeStyle = drawOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : drawOptions.color;
|
||||
ctx.fillStyle = drawOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : drawOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], parseInt(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
if (drawOptions.fillPolygons) {
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
function curves(ctx, points: number[] = []) {
|
||||
function curves(ctx, points: number[] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
if (!drawOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points);
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
return;
|
||||
}
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
|
|
@ -80,19 +122,20 @@ function curves(ctx, points: number[] = []) {
|
|||
}
|
||||
ctx.quadraticCurveTo(points[points.length - 2][0], points[points.length - 2][1], points[points.length - 1][0], points[points.length - 1][1]);
|
||||
ctx.stroke();
|
||||
if (drawOptions.fillPolygons) {
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
const ctx = inCanvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.font = drawOptions.font;
|
||||
ctx.fillStyle = drawOptions.color;
|
||||
ctx.font = localOptions.font;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
let i = 1;
|
||||
for (let j = 0; j < result.length; j++) {
|
||||
let where:any[] = [];
|
||||
|
|
@ -101,29 +144,30 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if ((what.length > 1) && (what[1].length > 0)) {
|
||||
const person = where[1] > 0 ? `#${where[1]}` : '';
|
||||
const label = `${where[0]} ${person}: ${what[1]}`;
|
||||
if (drawOptions.shadowColor && drawOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = drawOptions.shadowColor;
|
||||
ctx.fillText(label, 8, 2 + (i * drawOptions.lineHeight));
|
||||
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = localOptions.shadowColor;
|
||||
ctx.fillText(label, 8, 2 + (i * localOptions.lineHeight));
|
||||
}
|
||||
ctx.fillStyle = drawOptions.labelColor;
|
||||
ctx.fillText(label, 6, 0 + (i * drawOptions.lineHeight));
|
||||
ctx.fillStyle = localOptions.labelColor;
|
||||
ctx.fillText(label, 6, 0 + (i * localOptions.lineHeight));
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
const ctx = inCanvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
for (const f of result) {
|
||||
ctx.font = drawOptions.font;
|
||||
ctx.strokeStyle = drawOptions.color;
|
||||
ctx.fillStyle = drawOptions.color;
|
||||
if (drawOptions.drawBoxes) {
|
||||
if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3]);
|
||||
else rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]);
|
||||
ctx.font = localOptions.font;
|
||||
ctx.strokeStyle = localOptions.color;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
if (localOptions.drawBoxes) {
|
||||
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3], localOptions);
|
||||
else rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions);
|
||||
}
|
||||
// silly hack since fillText does not suport new line
|
||||
const labels:string[] = [];
|
||||
|
|
@ -138,24 +182,24 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
}
|
||||
if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`);
|
||||
if (labels.length === 0) labels.push('face');
|
||||
ctx.fillStyle = drawOptions.color;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
for (let i = labels.length - 1; i >= 0; i--) {
|
||||
const x = Math.max(f.box[0], 0);
|
||||
const y = i * drawOptions.lineHeight + f.box[1];
|
||||
if (drawOptions.shadowColor && drawOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = drawOptions.shadowColor;
|
||||
const y = i * localOptions.lineHeight + f.box[1];
|
||||
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = localOptions.shadowColor;
|
||||
ctx.fillText(labels[i], x + 5, y + 16);
|
||||
}
|
||||
ctx.fillStyle = drawOptions.labelColor;
|
||||
ctx.fillStyle = localOptions.labelColor;
|
||||
ctx.fillText(labels[i], x + 4, y + 15);
|
||||
}
|
||||
ctx.lineWidth = 1;
|
||||
if (f.mesh && f.mesh.length > 0) {
|
||||
if (drawOptions.drawPoints) {
|
||||
for (const pt of f.mesh) point(ctx, pt[0], pt[1], pt[2]);
|
||||
if (localOptions.drawPoints) {
|
||||
for (const pt of f.mesh) point(ctx, pt[0], pt[1], pt[2], localOptions);
|
||||
// for (const pt of f.meshRaw) point(ctx, pt[0] * inCanvas.offsetWidth, pt[1] * inCanvas.offsetHeight, pt[2]);
|
||||
}
|
||||
if (drawOptions.drawPolygons) {
|
||||
if (localOptions.drawPolygons) {
|
||||
ctx.lineWidth = 1;
|
||||
for (let i = 0; i < triangulation.length / 3; i++) {
|
||||
const points = [
|
||||
|
|
@ -163,30 +207,30 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
triangulation[i * 3 + 1],
|
||||
triangulation[i * 3 + 2],
|
||||
].map((index) => f.mesh[index]);
|
||||
lines(ctx, points);
|
||||
lines(ctx, points, localOptions);
|
||||
}
|
||||
// iris: array[center, left, top, right, bottom]
|
||||
if (f.annotations && f.annotations.leftEyeIris) {
|
||||
ctx.strokeStyle = drawOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : drawOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
|
||||
ctx.beginPath();
|
||||
const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2;
|
||||
ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
ctx.stroke();
|
||||
if (drawOptions.fillPolygons) {
|
||||
ctx.fillStyle = drawOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : drawOptions.color;
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (f.annotations && f.annotations.rightEyeIris) {
|
||||
ctx.strokeStyle = drawOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : drawOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
|
||||
ctx.beginPath();
|
||||
const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2;
|
||||
ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
ctx.stroke();
|
||||
if (drawOptions.fillPolygons) {
|
||||
ctx.fillStyle = drawOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : drawOptions.color;
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
|
@ -196,7 +240,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
}
|
||||
|
||||
const lastDrawnPose:any[] = [];
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
const ctx = inCanvas.getContext('2d');
|
||||
|
|
@ -204,31 +249,31 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
ctx.lineJoin = 'round';
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
// result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5);
|
||||
if (!lastDrawnPose[i] && drawOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] };
|
||||
ctx.strokeStyle = drawOptions.color;
|
||||
ctx.lineWidth = drawOptions.lineWidth;
|
||||
if (drawOptions.drawPoints) {
|
||||
if (!lastDrawnPose[i] && localOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] };
|
||||
ctx.strokeStyle = localOptions.color;
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
if (localOptions.drawPoints) {
|
||||
for (let pt = 0; pt < result[i].keypoints.length; pt++) {
|
||||
ctx.fillStyle = drawOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : drawOptions.color;
|
||||
if (drawOptions.bufferedOutput) {
|
||||
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : localOptions.color;
|
||||
if (localOptions.bufferedOutput) {
|
||||
lastDrawnPose[i].keypoints[pt][0] = (lastDrawnPose[i].keypoints[pt][0] + result[i].keypoints[pt].position.x) / 2;
|
||||
lastDrawnPose[i].keypoints[pt][1] = (lastDrawnPose[i].keypoints[pt][1] + result[i].keypoints[pt].position.y) / 2;
|
||||
point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1]);
|
||||
point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1], 0, localOptions);
|
||||
} else {
|
||||
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y);
|
||||
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (drawOptions.drawLabels) {
|
||||
ctx.font = drawOptions.font;
|
||||
if (localOptions.drawLabels) {
|
||||
ctx.font = localOptions.font;
|
||||
if (result[i].keypoints) {
|
||||
for (const pt of result[i].keypoints) {
|
||||
ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (drawOptions.drawPolygons && result[i].keypoints) {
|
||||
if (localOptions.drawPolygons && result[i].keypoints) {
|
||||
let part;
|
||||
const points: any[] = [];
|
||||
// shoulder line
|
||||
|
|
@ -237,7 +282,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
curves(ctx, points);
|
||||
curves(ctx, points, localOptions);
|
||||
// torso main
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
|
||||
|
|
@ -248,7 +293,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
if (points.length === 4) lines(ctx, points); // only draw if we have complete torso
|
||||
if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso
|
||||
// leg left
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftHip');
|
||||
|
|
@ -261,7 +306,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftFoot');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
curves(ctx, points);
|
||||
curves(ctx, points, localOptions);
|
||||
// leg right
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightHip');
|
||||
|
|
@ -274,7 +319,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightFoot');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
curves(ctx, points);
|
||||
curves(ctx, points, localOptions);
|
||||
// arm left
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
|
||||
|
|
@ -285,7 +330,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftPalm');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
curves(ctx, points);
|
||||
curves(ctx, points, localOptions);
|
||||
// arm right
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightShoulder');
|
||||
|
|
@ -296,50 +341,51 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
part = result[i].keypoints.find((a) => a.part === 'rightPalm');
|
||||
if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]);
|
||||
curves(ctx, points);
|
||||
curves(ctx, points, localOptions);
|
||||
// draw all
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
const ctx = inCanvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.lineJoin = 'round';
|
||||
ctx.font = drawOptions.font;
|
||||
ctx.font = localOptions.font;
|
||||
for (const h of result) {
|
||||
if (drawOptions.drawBoxes) {
|
||||
ctx.strokeStyle = drawOptions.color;
|
||||
ctx.fillStyle = drawOptions.color;
|
||||
if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]);
|
||||
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]);
|
||||
if (drawOptions.drawLabels) {
|
||||
if (drawOptions.shadowColor && drawOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = drawOptions.shadowColor;
|
||||
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + drawOptions.lineHeight, h.box[2]);
|
||||
if (localOptions.drawBoxes) {
|
||||
ctx.strokeStyle = localOptions.color;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions);
|
||||
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
|
||||
if (localOptions.drawLabels) {
|
||||
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = localOptions.shadowColor;
|
||||
ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]);
|
||||
}
|
||||
ctx.fillStyle = drawOptions.labelColor;
|
||||
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + drawOptions.lineHeight, h.box[2]);
|
||||
ctx.fillStyle = localOptions.labelColor;
|
||||
ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
if (drawOptions.drawPoints) {
|
||||
if (localOptions.drawPoints) {
|
||||
if (h.landmarks && h.landmarks.length > 0) {
|
||||
for (const pt of h.landmarks) {
|
||||
ctx.fillStyle = drawOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : drawOptions.color;
|
||||
point(ctx, pt[0], pt[1]);
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : localOptions.color;
|
||||
point(ctx, pt[0], pt[1], 0, localOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (drawOptions.drawPolygons) {
|
||||
if (localOptions.drawPolygons) {
|
||||
const addPart = (part) => {
|
||||
if (!part) return;
|
||||
for (let i = 0; i < part.length; i++) {
|
||||
ctx.lineWidth = drawOptions.lineWidth;
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = drawOptions.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : drawOptions.color;
|
||||
ctx.strokeStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : localOptions.color;
|
||||
ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]);
|
||||
ctx.lineTo(part[i][0], part[i][1]);
|
||||
ctx.stroke();
|
||||
|
|
@ -355,27 +401,28 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<any>) {
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
const ctx = inCanvas.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.lineJoin = 'round';
|
||||
ctx.font = drawOptions.font;
|
||||
ctx.font = localOptions.font;
|
||||
for (const h of result) {
|
||||
if (drawOptions.drawBoxes) {
|
||||
ctx.strokeStyle = drawOptions.color;
|
||||
ctx.fillStyle = drawOptions.color;
|
||||
if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]);
|
||||
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]);
|
||||
if (drawOptions.drawLabels) {
|
||||
if (localOptions.drawBoxes) {
|
||||
ctx.strokeStyle = localOptions.color;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions);
|
||||
else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions);
|
||||
if (localOptions.drawLabels) {
|
||||
const label = `${Math.round(100 * h.score)}% ${h.label}`;
|
||||
if (drawOptions.shadowColor && drawOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = drawOptions.shadowColor;
|
||||
ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + drawOptions.lineHeight, h.box[2]);
|
||||
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = localOptions.shadowColor;
|
||||
ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]);
|
||||
}
|
||||
ctx.fillStyle = drawOptions.labelColor;
|
||||
ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + drawOptions.lineHeight, h.box[2]);
|
||||
ctx.fillStyle = localOptions.labelColor;
|
||||
ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]);
|
||||
}
|
||||
ctx.stroke();
|
||||
}
|
||||
|
|
@ -389,12 +436,13 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE
|
|||
outCtx?.drawImage(inCanvas, 0, 0);
|
||||
}
|
||||
|
||||
export async function all(inCanvas: HTMLCanvasElement, result:any) {
|
||||
export async function all(inCanvas: HTMLCanvasElement, result:any, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
face(inCanvas, result.face);
|
||||
body(inCanvas, result.body);
|
||||
hand(inCanvas, result.hand);
|
||||
gesture(inCanvas, result.gesture);
|
||||
object(inCanvas, result.object);
|
||||
face(inCanvas, result.face, localOptions);
|
||||
body(inCanvas, result.body, localOptions);
|
||||
hand(inCanvas, result.hand, localOptions);
|
||||
gesture(inCanvas, result.gesture, localOptions);
|
||||
object(inCanvas, result.object, localOptions);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export function similarity(embedding1, embedding2, order = 2): number {
|
|||
if (embedding1?.length === 0 || embedding2?.length === 0) return 0;
|
||||
if (embedding1?.length !== embedding2?.length) return 0;
|
||||
// general minkowski distance, euclidean distance is limited case where order is 2
|
||||
const distance = 4.0 * embedding1
|
||||
const distance = 5.0 * embedding1
|
||||
.map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared
|
||||
.reduce((sum, now) => (sum + now), 0) // sum all distances
|
||||
** (1 / order); // get root of
|
||||
|
|
@ -51,6 +51,7 @@ export function enhance(input): Tensor {
|
|||
if (!(tensor instanceof tf.Tensor)) return null;
|
||||
// do a tight crop of image and resize it to fit the model
|
||||
const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right
|
||||
// const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test
|
||||
const crop = (tensor.shape.length === 3)
|
||||
? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing
|
||||
: tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]);
|
||||
|
|
@ -75,11 +76,13 @@ export function enhance(input): Tensor {
|
|||
const factor = 5;
|
||||
const contrast = merge.sub(mean).mul(factor).add(mean);
|
||||
*/
|
||||
|
||||
/*
|
||||
// normalize brightness from 0..1
|
||||
const darken = crop.sub(crop.min());
|
||||
const lighten = darken.div(darken.max());
|
||||
*/
|
||||
|
||||
const norm = crop.mul(255);
|
||||
|
||||
return norm;
|
||||
|
|
@ -96,9 +99,6 @@ export async function predict(image, config) {
|
|||
if (config.videoOptimized) skipped = 0;
|
||||
else skipped = Number.MAX_SAFE_INTEGER;
|
||||
return new Promise(async (resolve) => {
|
||||
// const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
// const enhanced = tf.mul(resize, [255.0]);
|
||||
// tf.dispose(resize);
|
||||
const enhanced = enhance(image);
|
||||
|
||||
let resT;
|
||||
|
|
|
|||
77
src/human.ts
77
src/human.ts
|
|
@ -25,15 +25,20 @@ import * as app from '../package.json';
|
|||
|
||||
/** Generic Tensor object type */
|
||||
export type Tensor = typeof tf.Tensor;
|
||||
|
||||
export type { Config } from './config';
|
||||
export type { Result } from './result';
|
||||
export type { DrawOptions } from './draw/draw';
|
||||
|
||||
/** Defines all possible input types for **Human** detection */
|
||||
export type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas;
|
||||
|
||||
/** Error message */
|
||||
export type Error = { error: string };
|
||||
|
||||
/** Instance of TensorFlow/JS */
|
||||
export type TensorFlow = typeof tf;
|
||||
|
||||
/** Generic Model object type, holds instance of individual models */
|
||||
type Model = Object;
|
||||
|
||||
|
|
@ -47,14 +52,32 @@ type Model = Object;
|
|||
* - Possible inputs: {@link Input}
|
||||
*/
|
||||
export class Human {
|
||||
/** Current version of Human library in semver format */
|
||||
version: string;
|
||||
/** Current configuration
|
||||
* - Details: {@link Config}
|
||||
*/
|
||||
config: Config;
|
||||
/** Current state of Human library
|
||||
* - Can be polled to determine operations that are currently executed
|
||||
*/
|
||||
state: string;
|
||||
image: { tensor: Tensor, canvas: OffscreenCanvas | HTMLCanvasElement };
|
||||
// classes
|
||||
/** Internal: Instance of current image being processed */
|
||||
image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
|
||||
/** Internal: Instance of TensorFlow/JS used by Human
|
||||
* - Can be embedded or externally provided
|
||||
*/
|
||||
tf: TensorFlow;
|
||||
/** Draw helper classes that can draw detected objects on canvas using specified draw styles
|
||||
* - options: global settings for all draw operations, can be overriden for each draw method, for details see {@link DrawOptions}
|
||||
* - face: draw detected faces
|
||||
* - body: draw detected people and body parts
|
||||
* - hand: draw detected hands and hand parts
|
||||
* - canvas: draw processed canvas which is a processed copy of the input
|
||||
* - all: meta-function that performs: canvas, face, body, hand
|
||||
*/
|
||||
draw: {
|
||||
drawOptions?: typeof draw.drawOptions,
|
||||
options: draw.DrawOptions,
|
||||
gesture: typeof draw.gesture,
|
||||
face: typeof draw.face,
|
||||
body: typeof draw.body,
|
||||
|
|
@ -62,7 +85,7 @@ export class Human {
|
|||
canvas: typeof draw.canvas,
|
||||
all: typeof draw.all,
|
||||
};
|
||||
// models
|
||||
/** Internal: Currently loaded models */
|
||||
models: {
|
||||
face: facemesh.MediaPipeFaceMesh | Model | null,
|
||||
posenet: posenet.PoseNet | null,
|
||||
|
|
@ -77,6 +100,7 @@ export class Human {
|
|||
nanodet: Model | null,
|
||||
faceres: Model | null,
|
||||
};
|
||||
/** Internal: Currently loaded classes */
|
||||
classes: {
|
||||
facemesh: typeof facemesh;
|
||||
age: typeof age;
|
||||
|
|
@ -87,16 +111,25 @@ export class Human {
|
|||
nanodet: typeof nanodet;
|
||||
faceres: typeof faceres;
|
||||
};
|
||||
/** Face triangualtion array of 468 points, used for triangle references between points */
|
||||
faceTriangulation: typeof facemesh.triangulation;
|
||||
/** UV map of 468 values, used for 3D mapping of the face mesh */
|
||||
faceUVMap: typeof facemesh.uvmap;
|
||||
/** Platform and agent information detected by Human */
|
||||
sysinfo: { platform: string, agent: string };
|
||||
/** Performance object that contains values for all recently performed operations */
|
||||
perf: any;
|
||||
#numTensors: number;
|
||||
#analyzeMemoryLeaks: boolean;
|
||||
#checkSanity: boolean;
|
||||
#firstRun: boolean;
|
||||
|
||||
// definition end
|
||||
|
||||
/**
|
||||
* Creates instance of Human library that is futher used for all operations
|
||||
* - @param userConfig: {@link Config}
|
||||
*/
|
||||
constructor(userConfig: Config | Object = {}) {
|
||||
this.tf = tf;
|
||||
this.draw = draw;
|
||||
|
|
@ -143,6 +176,9 @@ export class Human {
|
|||
this.sysinfo = sysinfo.info();
|
||||
}
|
||||
|
||||
/** Internal: ProfileData method returns last known profiling information
|
||||
* - Requires human.config.profile set to true
|
||||
*/
|
||||
profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} {
|
||||
if (this.config.profile) return profile.data;
|
||||
return {};
|
||||
|
|
@ -173,23 +209,39 @@ export class Human {
|
|||
return null;
|
||||
}
|
||||
|
||||
/** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings)
|
||||
* - Calculation is based on normalized Minkowski distance between
|
||||
*/
|
||||
similarity(embedding1: Array<number>, embedding2: Array<number>): number {
|
||||
if (this.config.face.description.enabled) return faceres.similarity(embedding1, embedding2);
|
||||
if (this.config.face.embedding.enabled) return embedding.similarity(embedding1, embedding2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Enhance method performs additional enhacements to face image previously detected for futher processing
|
||||
* @param input Tensor as provided in human.result.face[n].tensor
|
||||
* @returns Tensor
|
||||
*/
|
||||
// eslint-disable-next-line class-methods-use-this
|
||||
enhance(input: Tensor): Tensor | null {
|
||||
return faceres.enhance(input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Math method find best match between provided face descriptor and predefined database of known descriptors
|
||||
* @param faceEmbedding: face descriptor previsouly calculated on any face
|
||||
* @param db: array of mapping of face descriptors to known values
|
||||
* @param threshold: minimum score for matching to be considered in the result
|
||||
* @returns best match
|
||||
*/
|
||||
// eslint-disable-next-line class-methods-use-this
|
||||
match(faceEmbedding: Array<number>, db: Array<{ name: string, source: string, embedding: number[] }>, threshold = 0): { name: string, source: string, similarity: number, embedding: number[] } {
|
||||
return faceres.match(faceEmbedding, db, threshold);
|
||||
}
|
||||
|
||||
// preload models, not explicitly required as it's done automatically on first use
|
||||
/** Load method preloads all configured models on-demand
|
||||
* - Not explicitly required as any required model is load implicitly on it's first run
|
||||
*/
|
||||
async load(userConfig: Config | Object = {}) {
|
||||
this.state = 'load';
|
||||
const timeStamp = now();
|
||||
|
|
@ -261,7 +313,7 @@ export class Human {
|
|||
// check if backend needs initialization if it changed
|
||||
/** @hidden */
|
||||
#checkBackend = async (force = false) => {
|
||||
if (this.config.backend && (this.config.backend !== '') && force || (this.tf.getBackend() !== this.config.backend)) {
|
||||
if (this.config.backend && (this.config.backend.length > 0) && force || (this.tf.getBackend() !== this.config.backend)) {
|
||||
const timeStamp = now();
|
||||
this.state = 'backend';
|
||||
/* force backend reload
|
||||
|
|
@ -274,7 +326,7 @@ export class Human {
|
|||
}
|
||||
*/
|
||||
|
||||
if (this.config.backend && this.config.backend !== '') {
|
||||
if (this.config.backend && this.config.backend.length > 0) {
|
||||
// force browser vs node backend
|
||||
if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl';
|
||||
if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'wasm')) this.config.backend = 'tensorflow';
|
||||
|
|
@ -314,7 +366,12 @@ export class Human {
|
|||
}
|
||||
}
|
||||
|
||||
// main detect function
|
||||
/** Main detection method
|
||||
* - Analyze configuration: {@link Config}
|
||||
* - Pre-process input: {@link Input}
|
||||
* - Run inference for all configured models
|
||||
* - Process and return result: {@link Result}
|
||||
*/
|
||||
async detect(input: Input, userConfig: Config | Object = {}): Promise<Result | Error> {
|
||||
// detection happens inside a promise
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
@ -528,6 +585,10 @@ export class Human {
|
|||
return res;
|
||||
}
|
||||
|
||||
/** Warmup metho pre-initializes all models for faster inference
|
||||
* - can take significant time on startup
|
||||
* - only used for `webgl` and `humangl` backends
|
||||
*/
|
||||
async warmup(userConfig: Config | Object = {}): Promise<Result | { error }> {
|
||||
const t0 = now();
|
||||
if (userConfig) this.config = mergeDeep(this.config, userConfig);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export interface Result {
|
|||
* - emotion as array of possible emotions with their individual scores
|
||||
* - iris as distance value
|
||||
* - angle as object with values for roll, yaw and pitch angles
|
||||
* - tensor as Tensor object which contains detected face
|
||||
*/
|
||||
face: Array<{
|
||||
confidence: number,
|
||||
|
|
@ -44,6 +45,7 @@ export interface Result {
|
|||
angle: { roll: number, yaw: number, pitch: number },
|
||||
matrix: Array<[number, number, number, number, number, number, number, number, number]>
|
||||
}
|
||||
tensor: any,
|
||||
}>,
|
||||
/** Body results
|
||||
*
|
||||
|
|
@ -82,13 +84,12 @@ export interface Result {
|
|||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - part where gesture was detected
|
||||
* - gesture detected
|
||||
* - part: part name and number where gesture was detected: face, iris, body, hand
|
||||
* - gesture: gesture detected
|
||||
*/
|
||||
gesture: Array<{
|
||||
part: string,
|
||||
gesture: string,
|
||||
}>,
|
||||
gesture: Array<
|
||||
{ 'face': number, gesture: string } | { 'iris': number, gesture: string } | { 'body': number, gesture: string } | { 'hand': number, gesture: string }
|
||||
>,
|
||||
/** Object results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
|
|
|
|||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit bd0cfa7ff3eaf40cb114b45f5b16f88b9d213de8
|
||||
Subproject commit 77b1cd6cfd86abe0b21aae23e2be2beff84b68ff
|
||||
Loading…
Reference in New Issue