diff --git a/CHANGELOG.md b/CHANGELOG.md index b331e384..4023c447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **1.4.2** +Version: **1.5.0** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,6 +9,10 @@ Repository: **** ## Changelog +### **1.4.3** 2021/04/12 mandic00@live.com + +- implement webrtc + ### **1.4.2** 2021/04/12 mandic00@live.com - added support for multiple instances of human diff --git a/README.md b/README.md index 573085f8..93c06f40 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) fo - [**Code Repository**](https://github.com/vladmandic/human) - [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human) - [**Issues Tracker**](https://github.com/vladmandic/human/issues) -- [**API Specification: Human**](https://vladmandic.github.io/human/typedoc/classes/human.html) -- [**API Specification: Root**](https://vladmandic.github.io/human/typedoc/) +- [**TypeDoc API Specification: Human**](https://vladmandic.github.io/human/typedoc/classes/human.html) +- [**TypeDoc API Specification: Root**](https://vladmandic.github.io/human/typedoc/) - [**Change Log**](https://github.com/vladmandic/human/blob/main/CHANGELOG.md) ## Wiki pages diff --git a/demo/facematch.js b/demo/facematch.js index 8d81b767..7d30acf9 100644 --- a/demo/facematch.js +++ b/demo/facematch.js @@ -12,7 +12,7 @@ const userConfig = { enabled: true, detector: { rotation: true, return: true }, mesh: { enabled: true }, - embedding: { enabled: true }, + embedding: { enabled: false }, iris: { enabled: false }, age: { enabled: false }, gender: { enabled: false }, diff --git a/demo/index.js b/demo/index.js index 8efaeb79..45da4889 100644 --- a/demo/index.js +++ b/demo/index.js @@ -467,13 +467,13 @@ function setupMenu() { setupCamera(); }); menu.display.addHTML('
'); - menu.display.addBool('use 3D depth', human.draw.drawOptions, 'useDepth'); - menu.display.addBool('draw with curves', human.draw.drawOptions, 'useCurves'); - menu.display.addBool('print labels', human.draw.drawOptions, 'drawLabels'); - menu.display.addBool('draw points', human.draw.drawOptions, 'drawPoints'); - menu.display.addBool('draw boxes', human.draw.drawOptions, 'drawBoxes'); - menu.display.addBool('draw polygons', human.draw.drawOptions, 'drawPolygons'); - menu.display.addBool('fill polygons', human.draw.drawOptions, 'fillPolygons'); + menu.display.addBool('use 3D depth', human.draw.options, 'useDepth'); + menu.display.addBool('draw with curves', human.draw.options, 'useCurves'); + menu.display.addBool('print labels', human.draw.options, 'drawLabels'); + menu.display.addBool('draw points', human.draw.options, 'drawPoints'); + menu.display.addBool('draw boxes', human.draw.options, 'drawBoxes'); + menu.display.addBool('draw polygons', human.draw.options, 'drawPolygons'); + menu.display.addBool('fill polygons', human.draw.options, 'fillPolygons'); menu.image = new Menu(document.body, '', { top: `${document.getElementById('menubar').offsetHeight}px`, left: x[1] }); menu.image.addBool('enabled', human.config.filter, 'enabled', (val) => human.config.filter.enabled = val); diff --git a/package.json b/package.json index 4a86c4cb..7277c329 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vladmandic/human", - "version": "1.4.3", + "version": "1.5.0", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "sideEffects": false, "main": "dist/human.node.js", @@ -67,13 +67,14 @@ "@vladmandic/pilogger": "^0.2.16", "chokidar": "^3.5.1", "dayjs": "^1.10.4", - "esbuild": "^0.11.9", + "esbuild": "^0.11.10", "eslint": "^7.24.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^5.1.0", + "hint": "^6.1.3", "rimraf": "^3.0.2", "seedrandom": "^3.0.5", "simple-git": "^2.37.0", diff --git a/src/config.ts b/src/config.ts index 9acee2a1..fea6d0b6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,38 +7,99 @@ * Contains all configurable parameters */ export interface Config { - backend: string, + /** Backend used for TFJS operations */ + backend: null | '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow', + /** Path to *.wasm files if backend is set to `wasm` */ wasmPath: string, + /** Print debug statements to console */ debug: boolean, + /** Perform model loading and inference concurrently or sequentially */ async: boolean, + /** Collect and print profiling data during inference operations */ profile: boolean, + /** Internal: Use aggressive GPU memory deallocator when backend is set to `webgl` or `humangl` */ deallocate: boolean, + /** Internal: Run all inference operations in an explicit local scope run to avoid memory leaks */ scoped: boolean, + /** Perform additional optimizations when input is video, + * - must be disabled for images + * - automatically disabled for Image, ImageData, ImageBitmap and Tensor inputs + * - skips boundary detection for every `skipFrames` frames specified for each model + * - while maintaining in-box detection since objects don't change definition as fast */ videoOptimized: boolean, - warmup: string, + /** What to use for `human.warmup()` + * - warmup pre-initializes all models for faster inference but can take significant time on startup + * - only used for `webgl` and `humangl` backends + */ + warmup: 'none' | 'face' | 'full' | 'body', + /** Base model path (typically starting with file://, http:// or https://) for all models + * - individual modelPath values are joined to this path + */ modelBasePath: string, + /** Run input through image filters before inference + * - image filters run with near-zero latency as they are executed on the GPU + */ filter: { enabled: boolean, + /** Resize input width + * - if both width and height are set to 0, there is no resizing + * - if just one is set, second one is scaled automatically + * - if both are set, values are used as-is + */ width: number, + /** Resize input height + * - if both width and height are set to 0, there is no resizing + * - if just one is set, second one is scaled automatically + * - if both are set, values are used as-is + */ height: number, + /** Return processed canvas imagedata in result */ return: boolean, + /** Range: -1 (darken) to 1 (lighten) */ brightness: number, + /** Range: -1 (reduce contrast) to 1 (increase contrast) */ contrast: number, + /** Range: 0 (no sharpening) to 1 (maximum sharpening) */ sharpness: number, + /** Range: 0 (no blur) to N (blur radius in pixels) */ blur: number + /** Range: -1 (reduce saturation) to 1 (increase saturation) */ saturation: number, + /** Range: 0 (no change) to 360 (hue rotation in degrees) */ hue: number, + /** Image negative */ negative: boolean, + /** Image sepia colors */ sepia: boolean, + /** Image vintage colors */ vintage: boolean, + /** Image kodachrome colors */ kodachrome: boolean, + /** Image technicolor colors */ technicolor: boolean, + /** Image polaroid camera effect */ polaroid: boolean, + /** Range: 0 (no pixelate) to N (number of pixels to pixelate) */ pixelate: number, }, + /** Controlls gesture detection */ gesture: { enabled: boolean, }, + /** Controlls and configures all face-specific options: + * - face detection, face mesh detection, age, gender, emotion detection and face description + * Parameters: + * - enabled: true/false + * - modelPath: path for individual face model + * - rotation: use calculated rotated face image or just box with rotation as-is, false means higher performance, but incorrect mesh mapping on higher face angles + * - maxFaces: maximum number of faces detected in the input, should be set to the minimum number for performance + * - skipFrames: how many frames to go without re-running the face detector and just run modified face mesh analysis, only valid if videoOptimized is set to true + * - skipInitial: if previous detection resulted in no faces detected, should skipFrames be reset immediately to force new detection cycle + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression + * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression + * - return extracted face as tensor for futher user processing + */ face: { enabled: boolean, detector: { @@ -87,6 +148,13 @@ export interface Config { modelPath: string, }, }, + /** Controlls and configures all body detection specific options + * - enabled: true/false + * - modelPath: paths for both hand detector model and hand skeleton model + * - maxDetections: maximum number of people detected in the input, should be set to the minimum number for performance + * - scoreThreshold: threshold for deciding when to remove people based on score in non-maximum suppression + * - nmsRadius: threshold for deciding whether body parts overlap too much in non-maximum suppression + */ body: { enabled: boolean, modelPath: string, @@ -94,6 +162,18 @@ export interface Config { scoreThreshold: number, nmsRadius: number, }, + /** Controlls and configures all hand detection specific options + * - enabled: true/false + * - modelPath: paths for both hand detector model and hand skeleton model + * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted + * - skipFrames: how many frames to go without re-running the hand bounding box detector and just run modified hand skeleton detector, only valid if videoOptimized is set to true + * - skipInitial: if previous detection resulted in no hands detected, should skipFrames be reset immediately to force new detection cycle + * - minConfidence: threshold for discarding a prediction + * - iouThreshold: threshold for deciding whether boxes overlap too much in non-maximum suppression + * - scoreThreshold: threshold for deciding when to remove boxes based on score in non-maximum suppression + * - maxHands: maximum number of hands detected in the input, should be set to the minimum number for performance + * - landmarks: detect hand landmarks or just hand boundary box + */ hand: { enabled: boolean, rotation: boolean, @@ -111,6 +191,12 @@ export interface Config { modelPath: string, }, }, + /** Controlls and configures all object detection specific options + * - minConfidence: minimum score that detection must have to return as valid object + * - iouThreshold: ammount of overlap between two detected objects before one object is removed + * - maxResults: maximum number of detections to return + * - skipFrames: run object detection every n input frames, only valid if videoOptimized is set to true + */ object: { enabled: boolean, modelPath: string, @@ -134,16 +220,16 @@ const config: Config = { // this disables per-model performance data but // slightly increases performance // cannot be used if profiling is enabled - profile: false, // enable tfjs profiling + profile: false, // internal: enable tfjs profiling // this has significant performance impact // only enable for debugging purposes // currently only implemented for age,gender,emotion models - deallocate: false, // aggresively deallocate gpu memory after each usage - // only valid for webgl backend and only during first call + deallocate: false, // internal: aggresively deallocate gpu memory after each usage + // only valid for webgl and humangl backend and only during first call // cannot be changed unless library is reloaded // this has significant performance impact // only enable on low-memory devices - scoped: false, // enable scoped runs + scoped: false, // internal: enable scoped runs // some models *may* have memory leaks, // this wrapps everything in a local scope at a cost of performance // typically not needed @@ -155,7 +241,9 @@ const config: Config = { warmup: 'face', // what to use for human.warmup(), can be 'none', 'face', 'full' // warmup pre-initializes all models for faster inference but can take // significant time on startup - filter: { + // only used for `webgl` and `humangl` backends + filter: { // run input through image filters before inference + // image filters run with near-zero latency as they are executed on the GPU enabled: true, // enable image pre-processing filters width: 0, // resize input width height: 0, // resize input height @@ -201,7 +289,7 @@ const config: Config = { // box for updated face analysis as the head probably hasn't moved much // in short time (10 * 1/25 = 0.25 sec) skipInitial: false, // if previous detection resulted in no faces detected, - // should skipFrames be reset immediately + // should skipFrames be reset immediately to force new detection cycle minConfidence: 0.2, // threshold for discarding a prediction iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much in // non-maximum suppression (0.1 means drop if overlap 10%) @@ -289,8 +377,8 @@ const config: Config = { // e.g., if model is running st 25 FPS, we can re-use existing bounding // box for updated hand skeleton analysis as the hand probably // hasn't moved much in short time (10 * 1/25 = 0.25 sec) - skipInitial: false, // if previous detection resulted in no faces detected, - // should skipFrames be reset immediately + skipInitial: false, // if previous detection resulted in no hands detected, + // should skipFrames be reset immediately to force new detection cycle minConfidence: 0.1, // threshold for discarding a prediction iouThreshold: 0.1, // threshold for deciding whether boxes overlap too much // in non-maximum suppression diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 3842c54c..edc12d8e 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -1,7 +1,49 @@ import { defaults } from '../config'; import { TRI468 as triangulation } from '../blazeface/coords'; +import { mergeDeep } from '../helpers'; -export const drawOptions = { +/** + * Draw Options + * Accessed via `human.draw.options` or provided per each draw method as the drawOptions optional parameter + * -color: draw color + * -labelColor: color for labels + * -shadowColor: optional shadow color for labels + * -font: font for labels + * -lineHeight: line height for labels, used for multi-line labels, + * -lineWidth: width of any lines, + * -pointSize: size of any point, + * -roundRect: for boxes, round corners by this many pixels, + * -drawPoints: should points be drawn, + * -drawLabels: should labels be drawn, + * -drawBoxes: should boxes be drawn, + * -drawPolygons: should polygons be drawn, + * -fillPolygons: should drawn polygons be filled, + * -useDepth: use z-axis coordinate as color shade, + * -useCurves: draw polygons as cures or as lines, + * -bufferedOutput: experimental: allows to call draw methods multiple times for each detection and interpolate results between results thus achieving smoother animations + * -useRawBoxes: Boolean: internal: use non-normalized coordinates when performing draw methods, + */ +export interface DrawOptions { + color: string, + labelColor: string, + shadowColor: string, + font: string, + lineHeight: number, + lineWidth: number, + pointSize: number, + roundRect: number, + drawPoints: Boolean, + drawLabels: Boolean, + drawBoxes: Boolean, + drawPolygons: Boolean, + fillPolygons: Boolean, + useDepth: Boolean, + useCurves: Boolean, + bufferedOutput: Boolean, + useRawBoxes: Boolean, +} + +export const options: DrawOptions = { color: 'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel labelColor: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel shadowColor: 'black', @@ -21,55 +63,55 @@ export const drawOptions = { useRawBoxes: false, }; -function point(ctx, x, y, z = null) { - ctx.fillStyle = drawOptions.useDepth && z ? `rgba(${127.5 + (2 * (z || 0))}, ${127.5 - (2 * (z || 0))}, 255, 0.3)` : drawOptions.color; +function point(ctx, x, y, z = 0, localOptions) { + ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color; ctx.beginPath(); - ctx.arc(x, y, drawOptions.pointSize, 0, 2 * Math.PI); + ctx.arc(x, y, localOptions.pointSize, 0, 2 * Math.PI); ctx.fill(); } -function rect(ctx, x, y, width, height) { +function rect(ctx, x, y, width, height, localOptions) { ctx.beginPath(); - if (drawOptions.useCurves) { + if (localOptions.useCurves) { const cx = (x + x + width) / 2; const cy = (y + y + height) / 2; ctx.ellipse(cx, cy, width / 2, height / 2, 0, 0, 2 * Math.PI); } else { - ctx.lineWidth = drawOptions.lineWidth; - ctx.moveTo(x + drawOptions.roundRect, y); - ctx.lineTo(x + width - drawOptions.roundRect, y); - ctx.quadraticCurveTo(x + width, y, x + width, y + drawOptions.roundRect); - ctx.lineTo(x + width, y + height - drawOptions.roundRect); - ctx.quadraticCurveTo(x + width, y + height, x + width - drawOptions.roundRect, y + height); - ctx.lineTo(x + drawOptions.roundRect, y + height); - ctx.quadraticCurveTo(x, y + height, x, y + height - drawOptions.roundRect); - ctx.lineTo(x, y + drawOptions.roundRect); - ctx.quadraticCurveTo(x, y, x + drawOptions.roundRect, y); + ctx.lineWidth = localOptions.lineWidth; + ctx.moveTo(x + localOptions.roundRect, y); + ctx.lineTo(x + width - localOptions.roundRect, y); + ctx.quadraticCurveTo(x + width, y, x + width, y + localOptions.roundRect); + ctx.lineTo(x + width, y + height - localOptions.roundRect); + ctx.quadraticCurveTo(x + width, y + height, x + width - localOptions.roundRect, y + height); + ctx.lineTo(x + localOptions.roundRect, y + height); + ctx.quadraticCurveTo(x, y + height, x, y + height - localOptions.roundRect); + ctx.lineTo(x, y + localOptions.roundRect); + ctx.quadraticCurveTo(x, y, x + localOptions.roundRect, y); ctx.closePath(); } ctx.stroke(); } -function lines(ctx, points: number[] = []) { +function lines(ctx, points: number[] = [], localOptions) { if (points === undefined || points.length === 0) return; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); for (const pt of points) { - ctx.strokeStyle = drawOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : drawOptions.color; - ctx.fillStyle = drawOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : drawOptions.color; + ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; + ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; ctx.lineTo(pt[0], parseInt(pt[1])); } ctx.stroke(); - if (drawOptions.fillPolygons) { + if (localOptions.fillPolygons) { ctx.closePath(); ctx.fill(); } } -function curves(ctx, points: number[] = []) { +function curves(ctx, points: number[] = [], localOptions) { if (points === undefined || points.length === 0) return; - if (!drawOptions.useCurves || points.length <= 2) { - lines(ctx, points); + if (!localOptions.useCurves || points.length <= 2) { + lines(ctx, points, localOptions); return; } ctx.moveTo(points[0][0], points[0][1]); @@ -80,19 +122,20 @@ function curves(ctx, points: number[] = []) { } ctx.quadraticCurveTo(points[points.length - 2][0], points[points.length - 2][1], points[points.length - 1][0], points[points.length - 1][1]); ctx.stroke(); - if (drawOptions.fillPolygons) { + if (localOptions.fillPolygons) { ctx.closePath(); ctx.fill(); } } -export async function gesture(inCanvas: HTMLCanvasElement, result: Array) { +export async function gesture(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; - ctx.font = drawOptions.font; - ctx.fillStyle = drawOptions.color; + ctx.font = localOptions.font; + ctx.fillStyle = localOptions.color; let i = 1; for (let j = 0; j < result.length; j++) { let where:any[] = []; @@ -101,29 +144,30 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array) { if ((what.length > 1) && (what[1].length > 0)) { const person = where[1] > 0 ? `#${where[1]}` : ''; const label = `${where[0]} ${person}: ${what[1]}`; - if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { - ctx.fillStyle = drawOptions.shadowColor; - ctx.fillText(label, 8, 2 + (i * drawOptions.lineHeight)); + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(label, 8, 2 + (i * localOptions.lineHeight)); } - ctx.fillStyle = drawOptions.labelColor; - ctx.fillText(label, 6, 0 + (i * drawOptions.lineHeight)); + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(label, 6, 0 + (i * localOptions.lineHeight)); i += 1; } } } -export async function face(inCanvas: HTMLCanvasElement, result: Array) { +export async function face(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; for (const f of result) { - ctx.font = drawOptions.font; - ctx.strokeStyle = drawOptions.color; - ctx.fillStyle = drawOptions.color; - if (drawOptions.drawBoxes) { - if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3]); - else rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); + ctx.font = localOptions.font; + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + if (localOptions.drawBoxes) { + if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3], localOptions); + else rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions); } // silly hack since fillText does not suport new line const labels:string[] = []; @@ -138,24 +182,24 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array) { } if (f.rotation && f.rotation.angle && f.rotation.angle.roll) labels.push(`roll: ${Math.trunc(100 * f.rotation.angle.roll) / 100} yaw:${Math.trunc(100 * f.rotation.angle.yaw) / 100} pitch:${Math.trunc(100 * f.rotation.angle.pitch) / 100}`); if (labels.length === 0) labels.push('face'); - ctx.fillStyle = drawOptions.color; + ctx.fillStyle = localOptions.color; for (let i = labels.length - 1; i >= 0; i--) { const x = Math.max(f.box[0], 0); - const y = i * drawOptions.lineHeight + f.box[1]; - if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { - ctx.fillStyle = drawOptions.shadowColor; + const y = i * localOptions.lineHeight + f.box[1]; + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; ctx.fillText(labels[i], x + 5, y + 16); } - ctx.fillStyle = drawOptions.labelColor; + ctx.fillStyle = localOptions.labelColor; ctx.fillText(labels[i], x + 4, y + 15); } ctx.lineWidth = 1; if (f.mesh && f.mesh.length > 0) { - if (drawOptions.drawPoints) { - for (const pt of f.mesh) point(ctx, pt[0], pt[1], pt[2]); + if (localOptions.drawPoints) { + for (const pt of f.mesh) point(ctx, pt[0], pt[1], pt[2], localOptions); // for (const pt of f.meshRaw) point(ctx, pt[0] * inCanvas.offsetWidth, pt[1] * inCanvas.offsetHeight, pt[2]); } - if (drawOptions.drawPolygons) { + if (localOptions.drawPolygons) { ctx.lineWidth = 1; for (let i = 0; i < triangulation.length / 3; i++) { const points = [ @@ -163,30 +207,30 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array) { triangulation[i * 3 + 1], triangulation[i * 3 + 2], ].map((index) => f.mesh[index]); - lines(ctx, points); + lines(ctx, points, localOptions); } // iris: array[center, left, top, right, bottom] if (f.annotations && f.annotations.leftEyeIris) { - ctx.strokeStyle = drawOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : drawOptions.color; + ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; ctx.beginPath(); const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2; const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2; ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); - if (drawOptions.fillPolygons) { - ctx.fillStyle = drawOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : drawOptions.color; + if (localOptions.fillPolygons) { + ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; ctx.fill(); } } if (f.annotations && f.annotations.rightEyeIris) { - ctx.strokeStyle = drawOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : drawOptions.color; + ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; ctx.beginPath(); const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2; const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2; ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); ctx.stroke(); - if (drawOptions.fillPolygons) { - ctx.fillStyle = drawOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : drawOptions.color; + if (localOptions.fillPolygons) { + ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; ctx.fill(); } } @@ -196,7 +240,8 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array) { } const lastDrawnPose:any[] = []; -export async function body(inCanvas: HTMLCanvasElement, result: Array) { +export async function body(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); @@ -204,31 +249,31 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { ctx.lineJoin = 'round'; for (let i = 0; i < result.length; i++) { // result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5); - if (!lastDrawnPose[i] && drawOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; - ctx.strokeStyle = drawOptions.color; - ctx.lineWidth = drawOptions.lineWidth; - if (drawOptions.drawPoints) { + if (!lastDrawnPose[i] && localOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; + ctx.strokeStyle = localOptions.color; + ctx.lineWidth = localOptions.lineWidth; + if (localOptions.drawPoints) { for (let pt = 0; pt < result[i].keypoints.length; pt++) { - ctx.fillStyle = drawOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : drawOptions.color; - if (drawOptions.bufferedOutput) { + ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : localOptions.color; + if (localOptions.bufferedOutput) { lastDrawnPose[i].keypoints[pt][0] = (lastDrawnPose[i].keypoints[pt][0] + result[i].keypoints[pt].position.x) / 2; lastDrawnPose[i].keypoints[pt][1] = (lastDrawnPose[i].keypoints[pt][1] + result[i].keypoints[pt].position.y) / 2; - point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1]); + point(ctx, lastDrawnPose[i].keypoints[pt][0], lastDrawnPose[i].keypoints[pt][1], 0, localOptions); } else { - point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y); + point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); } } } - if (drawOptions.drawLabels) { - ctx.font = drawOptions.font; + if (localOptions.drawLabels) { + ctx.font = localOptions.font; if (result[i].keypoints) { for (const pt of result[i].keypoints) { - ctx.fillStyle = drawOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : drawOptions.color; + ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color; ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4); } } } - if (drawOptions.drawPolygons && result[i].keypoints) { + if (localOptions.drawPolygons && result[i].keypoints) { let part; const points: any[] = []; // shoulder line @@ -237,7 +282,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - curves(ctx, points); + curves(ctx, points, localOptions); // torso main points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); @@ -248,7 +293,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - if (points.length === 4) lines(ctx, points); // only draw if we have complete torso + if (points.length === 4) lines(ctx, points, localOptions); // only draw if we have complete torso // leg left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftHip'); @@ -261,7 +306,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftFoot'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - curves(ctx, points); + curves(ctx, points, localOptions); // leg right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightHip'); @@ -274,7 +319,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightFoot'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - curves(ctx, points); + curves(ctx, points, localOptions); // arm left points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); @@ -285,7 +330,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'leftPalm'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - curves(ctx, points); + curves(ctx, points, localOptions); // arm right points.length = 0; part = result[i].keypoints.find((a) => a.part === 'rightShoulder'); @@ -296,50 +341,51 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array) { if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); part = result[i].keypoints.find((a) => a.part === 'rightPalm'); if (part && part.score > defaults.body.scoreThreshold) points.push([part.position.x, part.position.y]); - curves(ctx, points); + curves(ctx, points, localOptions); // draw all } } } -export async function hand(inCanvas: HTMLCanvasElement, result: Array) { +export async function hand(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; ctx.lineJoin = 'round'; - ctx.font = drawOptions.font; + ctx.font = localOptions.font; for (const h of result) { - if (drawOptions.drawBoxes) { - ctx.strokeStyle = drawOptions.color; - ctx.fillStyle = drawOptions.color; - if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]); - else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); - if (drawOptions.drawLabels) { - if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { - ctx.fillStyle = drawOptions.shadowColor; - ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + drawOptions.lineHeight, h.box[2]); + if (localOptions.drawBoxes) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions); + else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions); + if (localOptions.drawLabels) { + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText('hand', h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); } - ctx.fillStyle = drawOptions.labelColor; - ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + drawOptions.lineHeight, h.box[2]); + ctx.fillStyle = localOptions.labelColor; + ctx.fillText('hand', h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); } ctx.stroke(); } - if (drawOptions.drawPoints) { + if (localOptions.drawPoints) { if (h.landmarks && h.landmarks.length > 0) { for (const pt of h.landmarks) { - ctx.fillStyle = drawOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : drawOptions.color; - point(ctx, pt[0], pt[1]); + ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : localOptions.color; + point(ctx, pt[0], pt[1], 0, localOptions); } } } - if (drawOptions.drawPolygons) { + if (localOptions.drawPolygons) { const addPart = (part) => { if (!part) return; for (let i = 0; i < part.length; i++) { - ctx.lineWidth = drawOptions.lineWidth; + ctx.lineWidth = localOptions.lineWidth; ctx.beginPath(); - ctx.strokeStyle = drawOptions.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : drawOptions.color; + ctx.strokeStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)` : localOptions.color; ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]); ctx.lineTo(part[i][0], part[i][1]); ctx.stroke(); @@ -355,27 +401,28 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array) { } } -export async function object(inCanvas: HTMLCanvasElement, result: Array) { +export async function object(inCanvas: HTMLCanvasElement, result: Array, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; const ctx = inCanvas.getContext('2d'); if (!ctx) return; ctx.lineJoin = 'round'; - ctx.font = drawOptions.font; + ctx.font = localOptions.font; for (const h of result) { - if (drawOptions.drawBoxes) { - ctx.strokeStyle = drawOptions.color; - ctx.fillStyle = drawOptions.color; - if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]); - else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); - if (drawOptions.drawLabels) { + if (localOptions.drawBoxes) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + if (localOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3], localOptions); + else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions); + if (localOptions.drawLabels) { const label = `${Math.round(100 * h.score)}% ${h.label}`; - if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { - ctx.fillStyle = drawOptions.shadowColor; - ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + drawOptions.lineHeight, h.box[2]); + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); } - ctx.fillStyle = drawOptions.labelColor; - ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + drawOptions.lineHeight, h.box[2]); + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); } ctx.stroke(); } @@ -389,12 +436,13 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE outCtx?.drawImage(inCanvas, 0, 0); } -export async function all(inCanvas: HTMLCanvasElement, result:any) { +export async function all(inCanvas: HTMLCanvasElement, result:any, drawOptions?: DrawOptions) { + const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; if (!(inCanvas instanceof HTMLCanvasElement)) return; - face(inCanvas, result.face); - body(inCanvas, result.body); - hand(inCanvas, result.hand); - gesture(inCanvas, result.gesture); - object(inCanvas, result.object); + face(inCanvas, result.face, localOptions); + body(inCanvas, result.body, localOptions); + hand(inCanvas, result.hand, localOptions); + gesture(inCanvas, result.gesture, localOptions); + object(inCanvas, result.object, localOptions); } diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 645b5582..bce1fad4 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -23,7 +23,7 @@ export function similarity(embedding1, embedding2, order = 2): number { if (embedding1?.length === 0 || embedding2?.length === 0) return 0; if (embedding1?.length !== embedding2?.length) return 0; // general minkowski distance, euclidean distance is limited case where order is 2 - const distance = 4.0 * embedding1 + const distance = 5.0 * embedding1 .map((val, i) => (Math.abs(embedding1[i] - embedding2[i]) ** order)) // distance squared .reduce((sum, now) => (sum + now), 0) // sum all distances ** (1 / order); // get root of @@ -51,6 +51,7 @@ export function enhance(input): Tensor { if (!(tensor instanceof tf.Tensor)) return null; // do a tight crop of image and resize it to fit the model const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right + // const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test const crop = (tensor.shape.length === 3) ? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); @@ -75,11 +76,13 @@ export function enhance(input): Tensor { const factor = 5; const contrast = merge.sub(mean).mul(factor).add(mean); */ + /* // normalize brightness from 0..1 const darken = crop.sub(crop.min()); const lighten = darken.div(darken.max()); */ + const norm = crop.mul(255); return norm; @@ -96,9 +99,6 @@ export async function predict(image, config) { if (config.videoOptimized) skipped = 0; else skipped = Number.MAX_SAFE_INTEGER; return new Promise(async (resolve) => { - // const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); - // const enhanced = tf.mul(resize, [255.0]); - // tf.dispose(resize); const enhanced = enhance(image); let resT; diff --git a/src/human.ts b/src/human.ts index 538b4f9c..9135580c 100644 --- a/src/human.ts +++ b/src/human.ts @@ -25,15 +25,20 @@ import * as app from '../package.json'; /** Generic Tensor object type */ export type Tensor = typeof tf.Tensor; + export type { Config } from './config'; export type { Result } from './result'; +export type { DrawOptions } from './draw/draw'; + /** Defines all possible input types for **Human** detection */ export type Input = Tensor | typeof Image | ImageData | ImageBitmap | HTMLImageElement | HTMLMediaElement | HTMLVideoElement | HTMLCanvasElement | OffscreenCanvas; /** Error message */ export type Error = { error: string }; + /** Instance of TensorFlow/JS */ export type TensorFlow = typeof tf; + /** Generic Model object type, holds instance of individual models */ type Model = Object; @@ -47,14 +52,32 @@ type Model = Object; * - Possible inputs: {@link Input} */ export class Human { + /** Current version of Human library in semver format */ version: string; + /** Current configuration + * - Details: {@link Config} + */ config: Config; + /** Current state of Human library + * - Can be polled to determine operations that are currently executed + */ state: string; - image: { tensor: Tensor, canvas: OffscreenCanvas | HTMLCanvasElement }; - // classes + /** Internal: Instance of current image being processed */ + image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null }; + /** Internal: Instance of TensorFlow/JS used by Human + * - Can be embedded or externally provided + */ tf: TensorFlow; + /** Draw helper classes that can draw detected objects on canvas using specified draw styles + * - options: global settings for all draw operations, can be overriden for each draw method, for details see {@link DrawOptions} + * - face: draw detected faces + * - body: draw detected people and body parts + * - hand: draw detected hands and hand parts + * - canvas: draw processed canvas which is a processed copy of the input + * - all: meta-function that performs: canvas, face, body, hand + */ draw: { - drawOptions?: typeof draw.drawOptions, + options: draw.DrawOptions, gesture: typeof draw.gesture, face: typeof draw.face, body: typeof draw.body, @@ -62,7 +85,7 @@ export class Human { canvas: typeof draw.canvas, all: typeof draw.all, }; - // models + /** Internal: Currently loaded models */ models: { face: facemesh.MediaPipeFaceMesh | Model | null, posenet: posenet.PoseNet | null, @@ -77,6 +100,7 @@ export class Human { nanodet: Model | null, faceres: Model | null, }; + /** Internal: Currently loaded classes */ classes: { facemesh: typeof facemesh; age: typeof age; @@ -87,16 +111,25 @@ export class Human { nanodet: typeof nanodet; faceres: typeof faceres; }; + /** Face triangualtion array of 468 points, used for triangle references between points */ faceTriangulation: typeof facemesh.triangulation; + /** UV map of 468 values, used for 3D mapping of the face mesh */ faceUVMap: typeof facemesh.uvmap; + /** Platform and agent information detected by Human */ sysinfo: { platform: string, agent: string }; + /** Performance object that contains values for all recently performed operations */ perf: any; #numTensors: number; #analyzeMemoryLeaks: boolean; #checkSanity: boolean; #firstRun: boolean; + // definition end + /** + * Creates instance of Human library that is futher used for all operations + * - @param userConfig: {@link Config} + */ constructor(userConfig: Config | Object = {}) { this.tf = tf; this.draw = draw; @@ -143,6 +176,9 @@ export class Human { this.sysinfo = sysinfo.info(); } + /** Internal: ProfileData method returns last known profiling information + * - Requires human.config.profile set to true + */ profileData(): { newBytes, newTensors, peakBytes, numKernelOps, timeKernelOps, slowestKernelOps, largestKernelOps } | {} { if (this.config.profile) return profile.data; return {}; @@ -173,23 +209,39 @@ export class Human { return null; } + /** Simmilarity method calculates simmilarity between two provided face descriptors (face embeddings) + * - Calculation is based on normalized Minkowski distance between + */ similarity(embedding1: Array, embedding2: Array): number { if (this.config.face.description.enabled) return faceres.similarity(embedding1, embedding2); if (this.config.face.embedding.enabled) return embedding.similarity(embedding1, embedding2); return 0; } + /** Enhance method performs additional enhacements to face image previously detected for futher processing + * @param input Tensor as provided in human.result.face[n].tensor + * @returns Tensor + */ // eslint-disable-next-line class-methods-use-this enhance(input: Tensor): Tensor | null { return faceres.enhance(input); } + /** + * Math method find best match between provided face descriptor and predefined database of known descriptors + * @param faceEmbedding: face descriptor previsouly calculated on any face + * @param db: array of mapping of face descriptors to known values + * @param threshold: minimum score for matching to be considered in the result + * @returns best match + */ // eslint-disable-next-line class-methods-use-this match(faceEmbedding: Array, db: Array<{ name: string, source: string, embedding: number[] }>, threshold = 0): { name: string, source: string, similarity: number, embedding: number[] } { return faceres.match(faceEmbedding, db, threshold); } - // preload models, not explicitly required as it's done automatically on first use + /** Load method preloads all configured models on-demand + * - Not explicitly required as any required model is load implicitly on it's first run + */ async load(userConfig: Config | Object = {}) { this.state = 'load'; const timeStamp = now(); @@ -261,7 +313,7 @@ export class Human { // check if backend needs initialization if it changed /** @hidden */ #checkBackend = async (force = false) => { - if (this.config.backend && (this.config.backend !== '') && force || (this.tf.getBackend() !== this.config.backend)) { + if (this.config.backend && (this.config.backend.length > 0) && force || (this.tf.getBackend() !== this.config.backend)) { const timeStamp = now(); this.state = 'backend'; /* force backend reload @@ -274,7 +326,7 @@ export class Human { } */ - if (this.config.backend && this.config.backend !== '') { + if (this.config.backend && this.config.backend.length > 0) { // force browser vs node backend if (this.tf.ENV.flags.IS_BROWSER && this.config.backend === 'tensorflow') this.config.backend = 'webgl'; if (this.tf.ENV.flags.IS_NODE && (this.config.backend === 'webgl' || this.config.backend === 'wasm')) this.config.backend = 'tensorflow'; @@ -314,7 +366,12 @@ export class Human { } } - // main detect function + /** Main detection method + * - Analyze configuration: {@link Config} + * - Pre-process input: {@link Input} + * - Run inference for all configured models + * - Process and return result: {@link Result} + */ async detect(input: Input, userConfig: Config | Object = {}): Promise { // detection happens inside a promise return new Promise(async (resolve) => { @@ -528,6 +585,10 @@ export class Human { return res; } + /** Warmup metho pre-initializes all models for faster inference + * - can take significant time on startup + * - only used for `webgl` and `humangl` backends + */ async warmup(userConfig: Config | Object = {}): Promise { const t0 = now(); if (userConfig) this.config = mergeDeep(this.config, userConfig); diff --git a/src/result.ts b/src/result.ts index 90625b5a..4902feaa 100644 --- a/src/result.ts +++ b/src/result.ts @@ -24,6 +24,7 @@ export interface Result { * - emotion as array of possible emotions with their individual scores * - iris as distance value * - angle as object with values for roll, yaw and pitch angles + * - tensor as Tensor object which contains detected face */ face: Array<{ confidence: number, @@ -44,6 +45,7 @@ export interface Result { angle: { roll: number, yaw: number, pitch: number }, matrix: Array<[number, number, number, number, number, number, number, number, number]> } + tensor: any, }>, /** Body results * @@ -82,13 +84,12 @@ export interface Result { * * Array of individual results with one object per detected gesture * Each result has: - * - part where gesture was detected - * - gesture detected + * - part: part name and number where gesture was detected: face, iris, body, hand + * - gesture: gesture detected */ - gesture: Array<{ - part: string, - gesture: string, - }>, + gesture: Array< + { 'face': number, gesture: string } | { 'iris': number, gesture: string } | { 'body': number, gesture: string } | { 'hand': number, gesture: string } + >, /** Object results * * Array of individual results with one object per detected gesture diff --git a/wiki b/wiki index bd0cfa7f..77b1cd6c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit bd0cfa7ff3eaf40cb114b45f5b16f88b9d213de8 +Subproject commit 77b1cd6cfd86abe0b21aae23e2be2beff84b68ff