From 8523da07b5eced675dcf13d68310ca136c3cb7cc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 17 Mar 2021 14:35:11 -0400 Subject: [PATCH] fix box clamping and raw output --- .gitignore | 1 + CHANGELOG.md | 7 +++++-- CODE_OF_CONDUCT | 21 +++++++++++++++++++++ CONTRIBUTING | 20 ++++++++++++++++++++ SECURITY.md | 4 ++++ human.service | 4 ++-- src/draw.ts | 11 +++++++---- src/handpose/handpose.ts | 16 +++++++++++----- src/human.ts | 13 ++++++++++++- src/nanodet/nanodet.ts | 8 ++++---- wiki | 2 +- 11 files changed, 88 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 7ddfe6d1..1d2e013f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules private +pnpm-lock.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d1e3fe..ce8f9a06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **1.1.7** +Version: **1.1.8** Description: **Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition** Author: **Vladimir Mandic ** @@ -11,9 +11,12 @@ Repository: **** ### **HEAD -> main** 2021/03/17 mandic00@live.com +- hierarchical readme notes -### **origin/main** 2021/03/17 mandic00@live.com +### **1.1.8** 2021/03/17 mandic00@live.com +- add experimental nanodet object detection +- full models signature - cleanup ### **1.1.7** 2021/03/16 mandic00@live.com diff --git a/CODE_OF_CONDUCT b/CODE_OF_CONDUCT index 55da76e8..dc9228f4 100644 --- a/CODE_OF_CONDUCT +++ b/CODE_OF_CONDUCT @@ -2,3 +2,24 @@
+Use your best judgement +If it will possibly make others uncomfortable, do not post it + +- Be respectful + Disagreement is not an opportunity to attack someone else's thoughts or opinions. Although views may differ, remember to approach every situation with patience and care +- Be considerate + Think about how your contribution will affect others in the community +- Be open minded + Embrace new people and new ideas. Our community is continually evolving and we welcome positive change + +Be mindful of your language +Any of the following behavior is unacceptable: + +- Offensive comments of any kind +- Threats or intimidation +- Sexually explicit material +- Or any other kinds of harassment + +If you believe someone is violating the code of conduct, we ask that you report it + +Participants asked to stop any harassing behavior are expected to comply immediately diff --git a/CONTRIBUTING b/CONTRIBUTING index 932f8e0c..22137a67 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -1,3 +1,23 @@ # Human Library: Contributing Guidelines
+ +Pull requests from everyone are welcome + +
+ +Procedure for contributing: +- Create a fork of the repository on github + In a top right corner of a GitHub, select "Fork" +- Clone your forked repository to your local system + `git clone https://github.com// +- Make your changes +- Test your changes against code guidelines + `npm run lint` +- Push changes to your fork +- Submit a PR (pull request) to `Human` + + +
+ +Your pull request will be reviewed and pending review results, merged into main branch diff --git a/SECURITY.md b/SECURITY.md index 9d41451d..8d6959d2 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,7 @@ # Human Library: Security Policy
+ +All issues are tracked publicly on GitHub + +Entire code base and indluded dependencies is automatically scanned against known security vulnerabilities diff --git a/human.service b/human.service index 08399b5f..6cf5d098 100644 --- a/human.service +++ b/human.service @@ -5,8 +5,8 @@ After=network.target network-online.target [Service] Type=simple Environment="NODE_ENV=production" -ExecStart=/home/vlado/.nvm/versions/node/v15.7.0/bin/node server/serve.js -WorkingDirectory=/home/vlado/dev/human +ExecStart= server/serve.js +WorkingDirectory= StandardOutput=inherit StandardError=inherit Restart=always diff --git a/src/draw.ts b/src/draw.ts index 9ee6bd75..ee6f1862 100644 --- a/src/draw.ts +++ b/src/draw.ts @@ -18,6 +18,7 @@ export const drawOptions = { useDepth: true, useCurves: false, bufferedOutput: false, + useRawBoxes: false, }; function point(ctx, x, y, z = null) { @@ -121,8 +122,8 @@ export async function face(inCanvas, result) { ctx.strokeStyle = drawOptions.color; ctx.fillStyle = drawOptions.color; if (drawOptions.drawBoxes) { - rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); - // rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3]); + if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * f.boxRaw[0], inCanvas.height * f.boxRaw[1], inCanvas.width * f.boxRaw[2], inCanvas.height * f.boxRaw[3]); + else rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3]); } // silly hack since fillText does not suport new line const labels:string[] = []; @@ -305,7 +306,8 @@ export async function hand(inCanvas, result) { if (drawOptions.drawBoxes) { ctx.strokeStyle = drawOptions.color; ctx.fillStyle = drawOptions.color; - rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); + if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]); + else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); if (drawOptions.drawLabels) { if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { ctx.fillStyle = drawOptions.shadowColor; @@ -357,7 +359,8 @@ export async function object(inCanvas, result) { if (drawOptions.drawBoxes) { ctx.strokeStyle = drawOptions.color; ctx.fillStyle = drawOptions.color; - rect(ctx, h.box[0], h.box[1], h.box[2] - h.box[0], h.box[3] - h.box[1]); + if (drawOptions.useRawBoxes) rect(ctx, inCanvas.width * h.boxRaw[0], inCanvas.height * h.boxRaw[1], inCanvas.width * h.boxRaw[2], inCanvas.height * h.boxRaw[3]); + else rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3]); if (drawOptions.drawLabels) { const label = `${Math.round(100 * h.score)}% ${h.label}`; if (drawOptions.shadowColor && drawOptions.shadowColor !== '') { diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 80b6f758..98ca38d0 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -29,7 +29,7 @@ export class HandPose { async estimateHands(input, config) { const predictions = await this.handPipeline.estimateHands(input, config); if (!predictions) return []; - const hands: Array<{ confidence: number, box: any, landmarks: any, annotations: any }> = []; + const hands: Array<{ confidence: number, box: any, boxRaw: any, landmarks: any, annotations: any }> = []; for (const prediction of predictions) { const annotations = {}; if (prediction.landmarks) { @@ -40,10 +40,16 @@ export class HandPose { const box = prediction.box ? [ Math.max(0, prediction.box.topLeft[0]), Math.max(0, prediction.box.topLeft[1]), - Math.min(input.shape[2], prediction.box.bottomRight[0]) - prediction.box.topLeft[0], - Math.min(input.shape[1], prediction.box.bottomRight[1]) - prediction.box.topLeft[1], - ] : 0; - hands.push({ confidence: prediction.confidence, box, landmarks: prediction.landmarks, annotations }); + Math.min(input.shape[2], prediction.box.bottomRight[0]) - Math.max(0, prediction.box.topLeft[0]), + Math.min(input.shape[1], prediction.box.bottomRight[1]) - Math.max(0, prediction.box.topLeft[1]), + ] : []; + const boxRaw = [ + (prediction.box.topLeft[0]) / input.shape[2], + (prediction.box.topLeft[1]) / input.shape[1], + (prediction.box.bottomRight[0] - prediction.box.topLeft[0]) / input.shape[2], + (prediction.box.bottomRight[1] - prediction.box.topLeft[1]) / input.shape[1], + ]; + hands.push({ confidence: prediction.confidence, box, boxRaw, landmarks: prediction.landmarks, annotations }); } return hands; } diff --git a/src/human.ts b/src/human.ts index 8ee31bd4..8b8d7297 100644 --- a/src/human.ts +++ b/src/human.ts @@ -54,6 +54,7 @@ export type Result = { hand: Array<{ confidence: Number, box: [Number, Number, Number, Number], + boxRaw: [Number, Number, Number, Number], landmarks: Array<[Number, Number, Number]>, annotations: Array<{ part: String, points: Array<[Number, Number, Number]>[] }>, }>, @@ -595,7 +596,17 @@ export class Human { this.#perf.total = Math.trunc(now() - timeStart); this.state = 'idle'; - resolve({ face: faceRes, body: bodyRes, hand: handRes, gesture: gestureRes, object: objectRes, performance: this.#perf, canvas: process.canvas }); + const result = { + face: faceRes, + body: bodyRes, + hand: handRes, + gesture: gestureRes, + object: objectRes, + performance: this.#perf, + canvas: process.canvas, + }; + // log('Result:', result); + resolve(result); }); } diff --git a/src/nanodet/nanodet.ts b/src/nanodet/nanodet.ts index 562519e6..38fb6f80 100644 --- a/src/nanodet/nanodet.ts +++ b/src/nanodet/nanodet.ts @@ -49,10 +49,10 @@ async function process(res, inputSize, outputShape, config) { ]; boxRaw = boxRaw.map((a) => Math.max(0, Math.min(a, 1))); // fix out-of-bounds coords const box = [ // results normalized to input image pixels - boxRaw[0] * outputShape[0], - boxRaw[1] * outputShape[1], - boxRaw[2] * outputShape[0], - boxRaw[3] * outputShape[1], + Math.max(0, (boxRaw[0] * outputShape[0])), + Math.max(0, (boxRaw[1] * outputShape[1])), + Math.min(1, (boxRaw[2] * outputShape[0]) - (boxRaw[0] * outputShape[0])), + Math.min(1, (boxRaw[3] * outputShape[1]) - (boxRaw[1] * outputShape[1])), ]; const result = { score: scoresMax[i], diff --git a/wiki b/wiki index 99c3d70e..91af84e9 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 99c3d70ec78fbbe0d12528c1d4b18ba1d27fef0d +Subproject commit 91af84e9543762c4f31be41dda15fb2a5549d8a6