From c60f442714b1b5887ae25feb35fa413bc9996402 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 4 Mar 2021 10:32:46 -0500 Subject: [PATCH] add blazepose and update readme --- Change-Log.md | 6 ++++- Configuration.md | 9 +++---- Credits.md | 1 + Demos.md | 4 +-- Models.md | 66 +++++++++++++++++++++++++++++++----------------- Outputs.md | 6 +++-- 6 files changed, 59 insertions(+), 33 deletions(-) diff --git a/Change-Log.md b/Change-Log.md index 26c806f..6a5151e 100644 --- a/Change-Log.md +++ b/Change-Log.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **0.30.6** +Version: **0.40.0** Description: **Human: AI-powered 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition** Author: **Vladimir Mandic ** @@ -11,6 +11,10 @@ Repository: **** ### **HEAD -> main** 2021/03/03 mandic00@live.com +- add todo list + +### **0.30.6** 2021/03/03 mandic00@live.com + - fine tuning age and face models ### **0.30.5** 2021/03/02 mandic00@live.com diff --git a/Configuration.md b/Configuration.md index 27eeb1c..454edb3 100644 --- a/Configuration.md +++ b/Configuration.md @@ -138,8 +138,7 @@ config = { age: { enabled: true, - modelPath: '../models/age-ssrnet-imdb.json', // can be 'age-ssrnet-imdb' or 'age-ssrnet-wiki' - // which determines training set for model + modelPath: '../models/age-ssrnet-imdb.json', inputSize: 64, // fixed value skipFrames: 31, // how many frames to go without re-running the detector // only used for video inputs @@ -148,7 +147,7 @@ config = { gender: { enabled: true, minConfidence: 0.1, // threshold for discarding a prediction - modelPath: '../models/gender.json', // can be 'gender', 'gender-ssrnet-imdb' or 'gender-ssrnet-wiki' + modelPath: '../models/gender.json', // can be 'gender' or 'gender-ssrnet-imdb' inputSize: 64, // fixed value skipFrames: 41, // how many frames to go without re-running the detector // only used for video inputs @@ -159,7 +158,7 @@ config = { inputSize: 64, // fixed value minConfidence: 0.2, // threshold for discarding a prediction skipFrames: 21, // how many frames to go without re-running the detector - modelPath: '../models/emotion-large.json', // can be 'mini', 'large' + modelPath: '../models/emotion.json', }, embedding: { @@ -169,7 +168,7 @@ config = { }, }, - body: { + posenet: { enabled: true, modelPath: '../models/posenet.json', inputSize: 257, // fixed value diff --git a/Credits.md b/Credits.md index d29e503..a044c08 100644 --- a/Credits.md +++ b/Credits.md @@ -5,6 +5,7 @@ - Facial Spacial Geometry: [**MediaPipe FaceMesh**](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view) - Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view) - Hand Detection & Skeleton: [**MediaPipe HandPose**](https://drive.google.com/file/d/1sv4sSb9BSNVZhLzxXJ0jBv9DqD-4jnAz/view) +- Body Pose Detection: [**BlazePose**](https://drive.google.com/file/d/10IU-DRP2ioSNjKFdiGbmmQX81xAYj88s/view) - Body Pose Detection: [**PoseNet**](https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5) - Age & Gender Prediction: [**SSR-Net**](https://github.com/shamangary/SSR-Net) - Emotion Prediction: [**Oarriaga**](https://github.com/oarriaga/face_classification) diff --git a/Demos.md b/Demos.md index 0727d15..be1af4a 100644 --- a/Demos.md +++ b/Demos.md @@ -103,7 +103,7 @@ found 0 vulnerabilities 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 121966 /models/iris.json ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 94507 /models/age-ssrnet-imdb.json ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 93349 /models/gender-ssrnet-imdb.json ::ffff:192.168.0.200 -2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 17954 /models/emotion-large.json ::ffff:192.168.0.200 +2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 17954 /models/emotion.json ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 48039 /models/posenet.json ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 128180 /models/handdetect.json ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/json 129067 /models/handskeleton.json ::ffff:192.168.0.200 @@ -114,7 +114,7 @@ found 0 vulnerabilities 2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 5032780 /models/posenet.bin ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 161236 /models/gender-ssrnet-imdb.bin ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 7031064 /models/handdetect.bin ::ffff:192.168.0.200 -2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 820516 /models/emotion-large.bin ::ffff:192.168.0.200 +2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 820516 /models/emotion.bin ::ffff:192.168.0.200 2020-12-12 10:12:55 DATA: GET/2.0 200 application/octet-stream 5502280 /models/handskeleton.bin ::ffff:192.168.0.200 ... ``` diff --git a/Models.md b/Models.md index c9a4a22..92e231f 100644 --- a/Models.md +++ b/Models.md @@ -1,4 +1,43 @@ -# List of all models included in Human library +# Models + +## Default models in Human library + +Default models in Human library are: + +- **Face Detection**: MediaPipe BlazeFace-Back +- **Face Mesh**: MediaPipe FaceMesh +- **Face Iris Analysis**: MediaPipe Iris +- **Emotion Detection**: Oarriaga Emotion +- **Gender Detection**: Oarriaga Gender +- **Age Detection**: SSR-Net Age IMDB +- **Body Analysis**: PoseNet +- **Face Embedding**: Sirius-AI MobileFaceNet Embedding + +## Notes + +**Face detection** can be switched from `BlazeFace` to `FaceBoxes` for use in NodeJS as `BlazeFace` is currently not supported in NodeJS + +Note that `FaceMesh` requires `BlazeFace` as their execution is performed in a single pipeline + +Additionally, there are two versions of `BlazeFace`: `back` optimized for detection of smaller faces and `front` optimized for detection of large faces + +
+ +**Body detection** can be switched from `PoseNet` to `BlazePose` depending on the use case: + +- `PoseNet`: Works with multiple people in frame, works with only partial people + Best described as works-anywhere, but not with great precision +- `BlazePose`: Works with single person in frame and that person should be fully visibile + But if conditions are met, it returns far more details (39 vs 17 keypoints) and is far more accurate + Furthermore, it returns 3D approximation of each point instead of 2D + +
+ +**Gender detection** can be switched from the default model to `SSR-Net` trained on IMDB dataset + +

+ +## List of all models included in Human library
@@ -8,36 +47,17 @@ | MediaPipe BlazeFace (Back) | 78K | blazeface-back.json | 527K | blazeface-back.bin | 112 | | MediaPipe FaceMesh | 88K | facemesh.json | 2.9M | facemesh.bin | 118 | | MediaPipe Iris | 120K | iris.json | 2.5M | iris.bin | 191 | -| Oarriaga Emotion (Mini) | 30K | emotion-mini.json | 220K | emotion-mini.bin | 39 | -| Oarriaga Emotion (Large) | 18K | emotion-large.json | 802K | emotion-large.bin | 23 | +| Oarriaga Emotion | 18K | emotion.json | 802K | emotion.bin | 23 | | SSR-Net Age (IMDB) | 93K | age-ssrnet-imdb.json | 158K | age-ssrnet-imdb.bin | 158 | -| SSR-Net Age (Wiki) | 93K | age-ssrnet-wiki.json | 158K | age-ssrnet-wiki.bin | 158 | | SSR-Net Gender (IMDB) | 92K | gender-ssrnet-imdb.json | 158K | gender-ssrnet-imdb.bin | 157 | -| SSR-Net Gender (Wiki) | 92K | gender-ssrnet-wiki.json | 158K | gender-ssrnet-wiki.bin | 157 | | Oarriaga Gender | 30K | gender.json | 198K | gender.bin | 39 | | PoseNet | 47K | posenet.json | 4.8M | posenet.bin | 62 | +| MediaPipe BlazePose | 158K | blazepose.json | 6.6M | blazepose.bin | 225 | | MediaPipe HandPose (HandDetect) | 126K | handdetect.json | 6.8M | handdetect.bin | 152 | | MediaPipe HandPose (HandSkeleton) | 127K | handskeleton.json | 5.3M | handskeleton.bin | 145 | | Sirius-AI MobileFaceNet | 125K | mobilefacenet.json | 5.0M | mobilefacenet.bin | 139 | +| FaceBoxes | 212K | faceboxes.json | 2.0M | faceboxes.bin | N/A |
*Note: All model definitions JSON files are parsed for human readability* - -
- -## Credits - -- Face Detection: [**MediaPipe BlazeFace**](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view) -- Facial Spacial Geometry: [**MediaPipe FaceMesh**](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view) -- Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view) -- Hand Detection & Skeleton: [**MediaPipe HandPose**](https://drive.google.com/file/d/1sv4sSb9BSNVZhLzxXJ0jBv9DqD-4jnAz/view) -- Body Pose Detection: [**PoseNet**](https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5) -- Age & Gender Prediction: [**SSR-Net**](https://github.com/shamangary/SSR-Net) -- Emotion Prediction: [**Oarriaga**](https://github.com/oarriaga/face_classification) -- Face Embedding: [**Sirius-AI MobileFaceNet**](https://github.com/sirius-ai/MobileFaceNet_TF) -- Image Filters: [**WebGLImageFilter**](https://github.com/phoboslab/WebGLImageFilter) -- Pinto Model Zoo: [**Pinto**](https://github.com/PINTO0309/PINTO_model_zoo) - -*Included models are included under license inherited from the original model source* -*Model code has substantially changed from source that it is considered a derivative work and not simple re-publishing* diff --git a/Outputs.md b/Outputs.md index 8dadb15..b6685e4 100644 --- a/Outputs.md +++ b/Outputs.md @@ -32,8 +32,10 @@ result = { body: // [ { - score, // , - keypoints, // 17 annotated landmarks + score, // , overal detection score, only used for 'posenet', not used for 'blazepose' + keypoints, // for 'posenet': 17 annotated landmarks + // for 'blazepose': 39 annotated landmarks + // presence denotes probability value in range 0..1 that the point is located within the frame } ], hand: //