From e060f1947a0ee8c9283cb4adc674573c879b0441 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 24 Apr 2021 11:49:26 -0400 Subject: [PATCH] update posenet model --- CHANGELOG.md | 10 ++- README.md | 2 +- demo/index.js | 2 +- package.json | 6 +- src/config.ts | 2 +- src/draw/draw.ts | 17 ++++- src/posenet/utils.ts | 154 +++++++++++++++++++++++++++++++++++++++++++ wiki | 2 +- 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 src/posenet/utils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ab5dafbc..9ede507b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,15 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/04/24 mandic00@live.com + + +### **origin/main** 2021/04/22 mandic00@live.com + + ### **1.6.1** 2021/04/22 mandic00@live.com - -### **origin/main** 2021/04/20 mandic00@live.com - +- add npmrc - added filter.flip feature - added demo load image from http - mobile demo optimization and iris gestures diff --git a/README.md b/README.md index 395da4ba..b6919535 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ Default models in Human library are: - **Face Description**: HSE FaceRes - **Face Iris Analysis**: MediaPipe Iris - **Emotion Detection**: Oarriaga Emotion -- **Body Analysis**: PoseNet +- **Body Analysis**: PoseNet (AtomicBits version) Note that alternative models are provided and can be enabled via configuration For example, `PoseNet` model can be switched for `BlazePose` model depending on the use case diff --git a/demo/index.js b/demo/index.js index 1f3a272d..3a0a9f59 100644 --- a/demo/index.js +++ b/demo/index.js @@ -29,7 +29,7 @@ const userConfig = { }, hand: { enabled: false }, gesture: { enabled: true }, - body: { enabled: false }, + body: { enabled: true, modelPath: 'posenet.json' }, // body: { enabled: true, modelPath: 'blazepose.json' }, // object: { enabled: true }, }; diff --git a/package.json b/package.json index 2f92bb1a..b591617a 100644 --- a/package.json +++ b/package.json @@ -68,8 +68,8 @@ "canvas": "^2.7.0", "chokidar": "^3.5.1", "dayjs": "^1.10.4", - "esbuild": "^0.11.12", - "eslint": "^7.24.0", + "esbuild": "^0.11.14", + "eslint": "^7.25.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-json": "^2.1.2", @@ -80,7 +80,7 @@ "seedrandom": "^3.0.5", "simple-git": "^2.38.0", "tslib": "^2.2.0", - "typedoc": "^0.20.35", + "typedoc": "^0.20.36", "typescript": "^4.2.4" } } diff --git a/src/config.ts b/src/config.ts index c71a7968..a0651214 100644 --- a/src/config.ts +++ b/src/config.ts @@ -363,7 +363,7 @@ const config: Config = { // can be either absolute path or relative to modelBasePath // can be 'posenet', 'blazepose' or 'efficientpose' // 'blazepose' and 'efficientpose' are experimental - maxDetections: 10, // maximum number of people detected in the input + maxDetections: 1, // maximum number of people detected in the input // should be set to the minimum number for performance // only valid for posenet as blazepose only detects single pose scoreThreshold: 0.3, // threshold for deciding when to remove boxes based on score diff --git a/src/draw/draw.ts b/src/draw/draw.ts index ed000911..f781e2ef 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -55,7 +55,7 @@ export const options: DrawOptions = { roundRect: 28, drawPoints: false, drawLabels: true, - drawBoxes: true, + drawBoxes: false, drawPolygons: true, fillPolygons: false, useDepth: true, @@ -253,7 +253,20 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw // result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5); if (!lastDrawnPose[i] && localOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] }; ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; ctx.lineWidth = localOptions.lineWidth; + ctx.font = localOptions.font; + if (localOptions.drawBoxes) { + rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions); + if (localOptions.drawLabels) { + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + } if (localOptions.drawPoints) { for (let pt = 0; pt < result[i].keypoints.length; pt++) { ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : localOptions.color; @@ -271,7 +284,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, draw if (result[i].keypoints) { for (const pt of result[i].keypoints) { ctx.fillStyle = localOptions.useDepth && pt.position.z ? `rgba(${127.5 + (2 * pt.position.z)}, ${127.5 - (2 * pt.position.z)}, 255, 0.5)` : localOptions.color; - ctx.fillText(`${pt.part}`, pt.position.x + 4, pt.position.y + 4); + ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position.x + 4, pt.position.y + 4); } } } diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts new file mode 100644 index 00000000..cf63df9b --- /dev/null +++ b/src/posenet/utils.ts @@ -0,0 +1,154 @@ +import * as kpt from './keypoints'; + +export function eitherPointDoesntMeetConfidence(a, b, minConfidence) { + return (a < minConfidence || b < minConfidence); +} + +export function getAdjacentKeyPoints(keypoints, minConfidence) { + return kpt.connectedPartIndices.reduce((result, [leftJoint, rightJoint]) => { + if (eitherPointDoesntMeetConfidence(keypoints[leftJoint].score, keypoints[rightJoint].score, minConfidence)) { + return result; + } + result.push([keypoints[leftJoint], keypoints[rightJoint]]); + return result; + }, []); +} + +export function getBoundingBox(keypoints) { + const coord = keypoints.reduce(({ maxX, maxY, minX, minY }, { position: { x, y } }) => ({ + maxX: Math.max(maxX, x), + maxY: Math.max(maxY, y), + minX: Math.min(minX, x), + minY: Math.min(minY, y), + }), { + maxX: Number.NEGATIVE_INFINITY, + maxY: Number.NEGATIVE_INFINITY, + minX: Number.POSITIVE_INFINITY, + minY: Number.POSITIVE_INFINITY, + }); + return [coord.minX, coord.minY, coord.maxX - coord.minX, coord.maxY - coord.minY]; +} + +export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) { + const scalePose = (pose, scaleY, scaleX) => ({ + score: pose.score, + box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)], + keypoints: pose.keypoints.map(({ score, part, position }) => ({ + score, + part, + position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) }, + })), + }); + + const scaledPoses = poses.map((pose) => scalePose(pose, height / inputResolutionHeight, width / inputResolutionWidth)); + return scaledPoses; +} + +// algorithm based on Coursera Lecture from Algorithms, Part 1: https://www.coursera.org/learn/algorithms-part1/lecture/ZjoSM/heapsort +export class MaxHeap { + priorityQueue: any; + numberOfElements: number; + getElementValue: any; + + constructor(maxSize, getElementValue) { + this.priorityQueue = new Array(maxSize); + this.numberOfElements = -1; + this.getElementValue = getElementValue; + } + + enqueue(x) { + this.priorityQueue[++this.numberOfElements] = x; + this.swim(this.numberOfElements); + } + + dequeue() { + const max = this.priorityQueue[0]; + this.exchange(0, this.numberOfElements--); + this.sink(0); + this.priorityQueue[this.numberOfElements + 1] = null; + return max; + } + + empty() { return this.numberOfElements === -1; } + + size() { return this.numberOfElements + 1; } + + all() { return this.priorityQueue.slice(0, this.numberOfElements + 1); } + + max() { return this.priorityQueue[0]; } + + swim(k) { + while (k > 0 && this.less(Math.floor(k / 2), k)) { + this.exchange(k, Math.floor(k / 2)); + k = Math.floor(k / 2); + } + } + + sink(k) { + while (2 * k <= this.numberOfElements) { + let j = 2 * k; + if (j < this.numberOfElements && this.less(j, j + 1)) j++; + if (!this.less(k, j)) break; + this.exchange(k, j); + k = j; + } + } + + getValueAt(i) { + return this.getElementValue(this.priorityQueue[i]); + } + + less(i, j) { + return this.getValueAt(i) < this.getValueAt(j); + } + + exchange(i, j) { + const t = this.priorityQueue[i]; + this.priorityQueue[i] = this.priorityQueue[j]; + this.priorityQueue[j] = t; + } +} + +export function getOffsetPoint(y, x, keypoint, offsets) { + return { + y: offsets.get(y, x, keypoint), + x: offsets.get(y, x, keypoint + kpt.count), + }; +} + +export function getImageCoords(part, outputStride, offsets) { + const { heatmapY, heatmapX, id: keypoint } = part; + const { y, x } = getOffsetPoint(heatmapY, heatmapX, keypoint, offsets); + return { + x: part.heatmapX * outputStride + x, + y: part.heatmapY * outputStride + y, + }; +} + +export function fillArray(element, size) { + const result = new Array(size); + for (let i = 0; i < size; i++) { + result[i] = element; + } + return result; +} + +export function clamp(a, min, max) { + if (a < min) return min; + if (a > max) return max; + return a; +} + +export function squaredDistance(y1, x1, y2, x2) { + const dy = y2 - y1; + const dx = x2 - x1; + return dy * dy + dx * dx; +} + +export function addVectors(a, b) { + return { x: a.x + b.x, y: a.y + b.y }; +} + +export function clampVector(a, min, max) { + return { y: clamp(a.y, min, max), x: clamp(a.x, min, max) }; +} diff --git a/wiki b/wiki index 4cb4534d..ee4cf3aa 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 4cb4534d2812d092946fbe51f8078df15b2c07d7 +Subproject commit ee4cf3aa27940b10e275ef9e8119e220c4b2d70d