mirror of https://github.com/vladmandic/human
work on blazepose
Vladimir Mandic
parent
77408fc2a7
commit
7396505391
10
config.js
10
config.js
|
|
@ -144,6 +144,16 @@ export default {
|
|||
modelType: 'MobileNet', // Human includes MobileNet version, but you can switch to ResNet
|
||||
},
|
||||
|
||||
pose: {
|
||||
enabled: false,
|
||||
scoreThreshold: 0.6, // threshold for deciding when to remove boxes based on score
|
||||
// in non-maximum suppression
|
||||
iouThreshold: 0.3, // threshold for deciding whether boxes overlap too much
|
||||
// in non-maximum suppression
|
||||
modelPath: '../models/blazepose.json',
|
||||
inputSize: 128, // fixed value
|
||||
},
|
||||
|
||||
hand: {
|
||||
enabled: true,
|
||||
rotation: false, // use best-guess rotated hand image or just box with rotation as-is
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@ import Menu from './menu.js';
|
|||
import GLBench from './gl-bench.js';
|
||||
|
||||
const userConfig = {}; // add any user configuration overrides
|
||||
|
||||
/*
|
||||
const userConfig = {
|
||||
face: { enabled: true, iris: { enabled: false } },
|
||||
body: { enabled: true },
|
||||
face: { enabled: false, iris: { enabled: false } },
|
||||
body: { enabled: false },
|
||||
hand: { enabled: false },
|
||||
pose: { enabled: true },
|
||||
};
|
||||
*/
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytes": 1905603,
|
||||
"bytes": 1912580,
|
||||
"imports": []
|
||||
},
|
||||
"demo/draw.js": {
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"demo/browser.js": {
|
||||
"bytes": 25470,
|
||||
"bytes": 25500,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/human.esm.js"
|
||||
|
|
@ -38,14 +38,14 @@
|
|||
"dist/demo-browser-index.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 2025768
|
||||
"bytes": 2040931
|
||||
},
|
||||
"dist/demo-browser-index.js": {
|
||||
"imports": [],
|
||||
"exports": [],
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 1898232
|
||||
"bytesInOutput": 1905205
|
||||
},
|
||||
"demo/draw.js": {
|
||||
"bytesInOutput": 7736
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
"bytesInOutput": 19424
|
||||
}
|
||||
},
|
||||
"bytes": 1951936
|
||||
"bytes": 1958909
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -16,7 +16,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"src/blazeface/blazeface.js": {
|
||||
"bytes": 7024,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -27,7 +27,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"src/blazeface/box.js": {
|
||||
"bytes": 1935,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -35,35 +35,35 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/util.js": {
|
||||
"src/blazeface/util.js": {
|
||||
"bytes": 3087,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/coords.js": {
|
||||
"src/blazeface/coords.js": {
|
||||
"bytes": 37915,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"src/blazeface/facepipeline.js": {
|
||||
"bytes": 14306,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/box.js"
|
||||
"path": "src/blazeface/box.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/util.js"
|
||||
"path": "src/blazeface/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/coords.js"
|
||||
"path": "src/blazeface/coords.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"src/blazeface/facemesh.js": {
|
||||
"bytes": 2991,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -73,13 +73,13 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/blazeface.js"
|
||||
"path": "src/blazeface/blazeface.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/facepipeline.js"
|
||||
"path": "src/blazeface/facepipeline.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/coords.js"
|
||||
"path": "src/blazeface/coords.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -147,7 +147,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/body/modelBase.js": {
|
||||
"src/posenet/modelBase.js": {
|
||||
"bytes": 1343,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -155,78 +155,78 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/body/heapSort.js": {
|
||||
"src/posenet/heapSort.js": {
|
||||
"bytes": 1590,
|
||||
"imports": []
|
||||
},
|
||||
"src/body/buildParts.js": {
|
||||
"src/posenet/buildParts.js": {
|
||||
"bytes": 1775,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/heapSort.js"
|
||||
"path": "src/posenet/heapSort.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/keypoints.js": {
|
||||
"src/posenet/keypoints.js": {
|
||||
"bytes": 2011,
|
||||
"imports": []
|
||||
},
|
||||
"src/body/vectors.js": {
|
||||
"src/posenet/vectors.js": {
|
||||
"bytes": 1273,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decoders.js": {
|
||||
"src/posenet/decoders.js": {
|
||||
"bytes": 2083,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decodePose.js": {
|
||||
"src/posenet/decodePose.js": {
|
||||
"bytes": 5368,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/vectors.js"
|
||||
"path": "src/posenet/vectors.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decoders.js"
|
||||
"path": "src/posenet/decoders.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decodeMultiple.js": {
|
||||
"src/posenet/decodeMultiple.js": {
|
||||
"bytes": 2373,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/buildParts.js"
|
||||
"path": "src/posenet/buildParts.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodePose.js"
|
||||
"path": "src/posenet/decodePose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/vectors.js"
|
||||
"path": "src/posenet/vectors.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/util.js": {
|
||||
"src/posenet/util.js": {
|
||||
"bytes": 2262,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/modelPoseNet.js": {
|
||||
"src/posenet/modelPoseNet.js": {
|
||||
"bytes": 2519,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -236,34 +236,45 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/modelBase.js"
|
||||
"path": "src/posenet/modelBase.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodeMultiple.js"
|
||||
"path": "src/posenet/decodeMultiple.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodePose.js"
|
||||
"path": "src/posenet/decodePose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/util.js"
|
||||
"path": "src/posenet/util.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/posenet.js": {
|
||||
"src/posenet/posenet.js": {
|
||||
"bytes": 712,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/modelPoseNet.js"
|
||||
"path": "src/posenet/modelPoseNet.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/util.js"
|
||||
"path": "src/posenet/util.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"src/blazepose/blazepose.js": {
|
||||
"bytes": 8999,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
},
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/handpose/box.js": {
|
||||
"bytes": 2522,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -271,43 +282,43 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 3548,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
"path": "src/handpose/box.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"src/handpose/util.js": {
|
||||
"bytes": 2346,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"src/handpose/handpipeline.js": {
|
||||
"bytes": 7246,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
"path": "src/handpose/box.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/util.js"
|
||||
"path": "src/handpose/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"src/handpose/anchors.js": {
|
||||
"bytes": 224151,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"src/handpose/handpose.js": {
|
||||
"bytes": 2578,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -317,13 +328,13 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handdetector.js"
|
||||
"path": "src/handpose/handdetector.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handpipeline.js"
|
||||
"path": "src/handpose/handpipeline.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/anchors.js"
|
||||
"path": "src/handpose/anchors.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -350,7 +361,7 @@
|
|||
]
|
||||
},
|
||||
"config.js": {
|
||||
"bytes": 9644,
|
||||
"bytes": 10062,
|
||||
"imports": []
|
||||
},
|
||||
"src/sample.js": {
|
||||
|
|
@ -362,7 +373,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 17544,
|
||||
"bytes": 17787,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -374,7 +385,7 @@
|
|||
"path": "src/tfjs/backend.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/facemesh.js"
|
||||
"path": "src/blazeface/facemesh.js"
|
||||
},
|
||||
{
|
||||
"path": "src/age/age.js"
|
||||
|
|
@ -389,10 +400,13 @@
|
|||
"path": "src/embedding/embedding.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/posenet.js"
|
||||
"path": "src/posenet/posenet.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handpose.js"
|
||||
"path": "src/blazepose/blazepose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/handpose/handpose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/gesture/gesture.js"
|
||||
|
|
@ -419,7 +433,7 @@
|
|||
"dist/human.esm.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 1930371
|
||||
"bytes": 1945558
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"imports": [],
|
||||
|
|
@ -427,83 +441,83 @@
|
|||
"default"
|
||||
],
|
||||
"inputs": {
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 5039
|
||||
"src/blazeface/blazeface.js": {
|
||||
"bytesInOutput": 5046
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"src/blazeface/box.js": {
|
||||
"bytesInOutput": 1567
|
||||
},
|
||||
"src/face/util.js": {
|
||||
"src/blazeface/util.js": {
|
||||
"bytesInOutput": 2418
|
||||
},
|
||||
"src/face/coords.js": {
|
||||
"src/blazeface/coords.js": {
|
||||
"bytesInOutput": 30718
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"src/blazeface/facepipeline.js": {
|
||||
"bytesInOutput": 9386
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"src/blazeface/facemesh.js": {
|
||||
"bytesInOutput": 2322
|
||||
},
|
||||
"src/profile.js": {
|
||||
"bytesInOutput": 851
|
||||
},
|
||||
"src/age/age.js": {
|
||||
"bytesInOutput": 1186
|
||||
"bytesInOutput": 1193
|
||||
},
|
||||
"src/gender/gender.js": {
|
||||
"bytesInOutput": 1914
|
||||
"bytesInOutput": 1922
|
||||
},
|
||||
"src/emotion/emotion.js": {
|
||||
"bytesInOutput": 1813
|
||||
"bytesInOutput": 1820
|
||||
},
|
||||
"src/embedding/embedding.js": {
|
||||
"bytesInOutput": 1323
|
||||
"bytesInOutput": 1330
|
||||
},
|
||||
"src/body/modelBase.js": {
|
||||
"src/posenet/modelBase.js": {
|
||||
"bytesInOutput": 1055
|
||||
},
|
||||
"src/body/heapSort.js": {
|
||||
"src/posenet/heapSort.js": {
|
||||
"bytesInOutput": 1144
|
||||
},
|
||||
"src/body/buildParts.js": {
|
||||
"src/posenet/buildParts.js": {
|
||||
"bytesInOutput": 1287
|
||||
},
|
||||
"src/body/keypoints.js": {
|
||||
"src/posenet/keypoints.js": {
|
||||
"bytesInOutput": 1813
|
||||
},
|
||||
"src/body/vectors.js": {
|
||||
"src/posenet/vectors.js": {
|
||||
"bytesInOutput": 1055
|
||||
},
|
||||
"src/body/decoders.js": {
|
||||
"src/posenet/decoders.js": {
|
||||
"bytesInOutput": 1709
|
||||
},
|
||||
"src/body/decodePose.js": {
|
||||
"src/posenet/decodePose.js": {
|
||||
"bytesInOutput": 4098
|
||||
},
|
||||
"src/body/decodeMultiple.js": {
|
||||
"src/posenet/decodeMultiple.js": {
|
||||
"bytesInOutput": 1643
|
||||
},
|
||||
"src/body/util.js": {
|
||||
"src/posenet/util.js": {
|
||||
"bytesInOutput": 1892
|
||||
},
|
||||
"src/body/modelPoseNet.js": {
|
||||
"src/posenet/modelPoseNet.js": {
|
||||
"bytesInOutput": 1997
|
||||
},
|
||||
"src/body/posenet.js": {
|
||||
"src/posenet/posenet.js": {
|
||||
"bytesInOutput": 627
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 2747
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"src/handpose/handpipeline.js": {
|
||||
"bytesInOutput": 4484
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"src/handpose/anchors.js": {
|
||||
"bytesInOutput": 127037
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytesInOutput": 2023
|
||||
"src/handpose/handpose.js": {
|
||||
"bytesInOutput": 2025
|
||||
},
|
||||
"src/gesture/gesture.js": {
|
||||
"bytesInOutput": 3162
|
||||
|
|
@ -524,16 +538,19 @@
|
|||
"bytesInOutput": 918
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 11286
|
||||
"bytesInOutput": 11921
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"src/blazepose/blazepose.js": {
|
||||
"bytesInOutput": 6201
|
||||
},
|
||||
"src/handpose/box.js": {
|
||||
"bytesInOutput": 1420
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"src/handpose/util.js": {
|
||||
"bytesInOutput": 1796
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1514
|
||||
"bytesInOutput": 1617
|
||||
},
|
||||
"src/sample.js": {
|
||||
"bytesInOutput": 55299
|
||||
|
|
@ -542,7 +559,7 @@
|
|||
"bytesInOutput": 23
|
||||
}
|
||||
},
|
||||
"bytes": 1905603
|
||||
"bytes": 1912580
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -16,7 +16,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/blazeface.js": {
|
||||
"src/blazeface/blazeface.js": {
|
||||
"bytes": 7024,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -27,7 +27,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"src/blazeface/box.js": {
|
||||
"bytes": 1935,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -35,35 +35,35 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/face/util.js": {
|
||||
"src/blazeface/util.js": {
|
||||
"bytes": 3087,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/coords.js": {
|
||||
"src/blazeface/coords.js": {
|
||||
"bytes": 37915,
|
||||
"imports": []
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"src/blazeface/facepipeline.js": {
|
||||
"bytes": 14306,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/box.js"
|
||||
"path": "src/blazeface/box.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/util.js"
|
||||
"path": "src/blazeface/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/coords.js"
|
||||
"path": "src/blazeface/coords.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"src/blazeface/facemesh.js": {
|
||||
"bytes": 2991,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -73,13 +73,13 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/blazeface.js"
|
||||
"path": "src/blazeface/blazeface.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/facepipeline.js"
|
||||
"path": "src/blazeface/facepipeline.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/coords.js"
|
||||
"path": "src/blazeface/coords.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -147,7 +147,7 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/body/modelBase.js": {
|
||||
"src/posenet/modelBase.js": {
|
||||
"bytes": 1343,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -155,78 +155,78 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/body/heapSort.js": {
|
||||
"src/posenet/heapSort.js": {
|
||||
"bytes": 1590,
|
||||
"imports": []
|
||||
},
|
||||
"src/body/buildParts.js": {
|
||||
"src/posenet/buildParts.js": {
|
||||
"bytes": 1775,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/heapSort.js"
|
||||
"path": "src/posenet/heapSort.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/keypoints.js": {
|
||||
"src/posenet/keypoints.js": {
|
||||
"bytes": 2011,
|
||||
"imports": []
|
||||
},
|
||||
"src/body/vectors.js": {
|
||||
"src/posenet/vectors.js": {
|
||||
"bytes": 1273,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decoders.js": {
|
||||
"src/posenet/decoders.js": {
|
||||
"bytes": 2083,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decodePose.js": {
|
||||
"src/posenet/decodePose.js": {
|
||||
"bytes": 5368,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/vectors.js"
|
||||
"path": "src/posenet/vectors.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decoders.js"
|
||||
"path": "src/posenet/decoders.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/decodeMultiple.js": {
|
||||
"src/posenet/decodeMultiple.js": {
|
||||
"bytes": 2373,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/buildParts.js"
|
||||
"path": "src/posenet/buildParts.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodePose.js"
|
||||
"path": "src/posenet/decodePose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/vectors.js"
|
||||
"path": "src/posenet/vectors.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/util.js": {
|
||||
"src/posenet/util.js": {
|
||||
"bytes": 2262,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/modelPoseNet.js": {
|
||||
"src/posenet/modelPoseNet.js": {
|
||||
"bytes": 2519,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -236,34 +236,45 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/modelBase.js"
|
||||
"path": "src/posenet/modelBase.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodeMultiple.js"
|
||||
"path": "src/posenet/decodeMultiple.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/decodePose.js"
|
||||
"path": "src/posenet/decodePose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/util.js"
|
||||
"path": "src/posenet/util.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/body/posenet.js": {
|
||||
"src/posenet/posenet.js": {
|
||||
"bytes": 712,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/body/modelPoseNet.js"
|
||||
"path": "src/posenet/modelPoseNet.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/keypoints.js"
|
||||
"path": "src/posenet/keypoints.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/util.js"
|
||||
"path": "src/posenet/util.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"src/blazepose/blazepose.js": {
|
||||
"bytes": 8999,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
},
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/handpose/box.js": {
|
||||
"bytes": 2522,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -271,43 +282,43 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytes": 3548,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
"path": "src/handpose/box.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"src/handpose/util.js": {
|
||||
"bytes": 2346,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"src/handpose/handpipeline.js": {
|
||||
"bytes": 7246,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/box.js"
|
||||
"path": "src/handpose/box.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/util.js"
|
||||
"path": "src/handpose/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"src/handpose/anchors.js": {
|
||||
"bytes": 224151,
|
||||
"imports": []
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"src/handpose/handpose.js": {
|
||||
"bytes": 2578,
|
||||
"imports": [
|
||||
{
|
||||
|
|
@ -317,13 +328,13 @@
|
|||
"path": "dist/tfjs.esm.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handdetector.js"
|
||||
"path": "src/handpose/handdetector.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handpipeline.js"
|
||||
"path": "src/handpose/handpipeline.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/anchors.js"
|
||||
"path": "src/handpose/anchors.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -350,7 +361,7 @@
|
|||
]
|
||||
},
|
||||
"config.js": {
|
||||
"bytes": 9644,
|
||||
"bytes": 10062,
|
||||
"imports": []
|
||||
},
|
||||
"src/sample.js": {
|
||||
|
|
@ -362,7 +373,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 17544,
|
||||
"bytes": 17787,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -374,7 +385,7 @@
|
|||
"path": "src/tfjs/backend.js"
|
||||
},
|
||||
{
|
||||
"path": "src/face/facemesh.js"
|
||||
"path": "src/blazeface/facemesh.js"
|
||||
},
|
||||
{
|
||||
"path": "src/age/age.js"
|
||||
|
|
@ -389,10 +400,13 @@
|
|||
"path": "src/embedding/embedding.js"
|
||||
},
|
||||
{
|
||||
"path": "src/body/posenet.js"
|
||||
"path": "src/posenet/posenet.js"
|
||||
},
|
||||
{
|
||||
"path": "src/hand/handpose.js"
|
||||
"path": "src/blazepose/blazepose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/handpose/handpose.js"
|
||||
},
|
||||
{
|
||||
"path": "src/gesture/gesture.js"
|
||||
|
|
@ -419,89 +433,89 @@
|
|||
"dist/human.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 1930388
|
||||
"bytes": 1945575
|
||||
},
|
||||
"dist/human.js": {
|
||||
"imports": [],
|
||||
"exports": [],
|
||||
"inputs": {
|
||||
"src/face/blazeface.js": {
|
||||
"bytesInOutput": 5039
|
||||
"src/blazeface/blazeface.js": {
|
||||
"bytesInOutput": 5046
|
||||
},
|
||||
"src/face/box.js": {
|
||||
"src/blazeface/box.js": {
|
||||
"bytesInOutput": 1567
|
||||
},
|
||||
"src/face/util.js": {
|
||||
"src/blazeface/util.js": {
|
||||
"bytesInOutput": 2418
|
||||
},
|
||||
"src/face/coords.js": {
|
||||
"src/blazeface/coords.js": {
|
||||
"bytesInOutput": 30718
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"src/blazeface/facepipeline.js": {
|
||||
"bytesInOutput": 9386
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"src/blazeface/facemesh.js": {
|
||||
"bytesInOutput": 2322
|
||||
},
|
||||
"src/profile.js": {
|
||||
"bytesInOutput": 851
|
||||
},
|
||||
"src/age/age.js": {
|
||||
"bytesInOutput": 1186
|
||||
"bytesInOutput": 1193
|
||||
},
|
||||
"src/gender/gender.js": {
|
||||
"bytesInOutput": 1914
|
||||
"bytesInOutput": 1922
|
||||
},
|
||||
"src/emotion/emotion.js": {
|
||||
"bytesInOutput": 1813
|
||||
"bytesInOutput": 1820
|
||||
},
|
||||
"src/embedding/embedding.js": {
|
||||
"bytesInOutput": 1323
|
||||
"bytesInOutput": 1330
|
||||
},
|
||||
"src/body/modelBase.js": {
|
||||
"src/posenet/modelBase.js": {
|
||||
"bytesInOutput": 1055
|
||||
},
|
||||
"src/body/heapSort.js": {
|
||||
"src/posenet/heapSort.js": {
|
||||
"bytesInOutput": 1144
|
||||
},
|
||||
"src/body/buildParts.js": {
|
||||
"src/posenet/buildParts.js": {
|
||||
"bytesInOutput": 1287
|
||||
},
|
||||
"src/body/keypoints.js": {
|
||||
"src/posenet/keypoints.js": {
|
||||
"bytesInOutput": 1813
|
||||
},
|
||||
"src/body/vectors.js": {
|
||||
"src/posenet/vectors.js": {
|
||||
"bytesInOutput": 1055
|
||||
},
|
||||
"src/body/decoders.js": {
|
||||
"src/posenet/decoders.js": {
|
||||
"bytesInOutput": 1709
|
||||
},
|
||||
"src/body/decodePose.js": {
|
||||
"src/posenet/decodePose.js": {
|
||||
"bytesInOutput": 4098
|
||||
},
|
||||
"src/body/decodeMultiple.js": {
|
||||
"src/posenet/decodeMultiple.js": {
|
||||
"bytesInOutput": 1643
|
||||
},
|
||||
"src/body/util.js": {
|
||||
"src/posenet/util.js": {
|
||||
"bytesInOutput": 1892
|
||||
},
|
||||
"src/body/modelPoseNet.js": {
|
||||
"src/posenet/modelPoseNet.js": {
|
||||
"bytesInOutput": 1997
|
||||
},
|
||||
"src/body/posenet.js": {
|
||||
"src/posenet/posenet.js": {
|
||||
"bytesInOutput": 627
|
||||
},
|
||||
"src/hand/handdetector.js": {
|
||||
"src/handpose/handdetector.js": {
|
||||
"bytesInOutput": 2747
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"src/handpose/handpipeline.js": {
|
||||
"bytesInOutput": 4484
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"src/handpose/anchors.js": {
|
||||
"bytesInOutput": 127037
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytesInOutput": 2023
|
||||
"src/handpose/handpose.js": {
|
||||
"bytesInOutput": 2025
|
||||
},
|
||||
"src/gesture/gesture.js": {
|
||||
"bytesInOutput": 3162
|
||||
|
|
@ -513,7 +527,7 @@
|
|||
"bytesInOutput": 3652
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytesInOutput": 11351
|
||||
"bytesInOutput": 11986
|
||||
},
|
||||
"src/log.js": {
|
||||
"bytesInOutput": 266
|
||||
|
|
@ -524,14 +538,17 @@
|
|||
"src/tfjs/backend.js": {
|
||||
"bytesInOutput": 918
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"src/blazepose/blazepose.js": {
|
||||
"bytesInOutput": 6201
|
||||
},
|
||||
"src/handpose/box.js": {
|
||||
"bytesInOutput": 1420
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"src/handpose/util.js": {
|
||||
"bytesInOutput": 1796
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1514
|
||||
"bytesInOutput": 1617
|
||||
},
|
||||
"src/sample.js": {
|
||||
"bytesInOutput": 55299
|
||||
|
|
@ -540,7 +557,7 @@
|
|||
"bytesInOutput": 23
|
||||
}
|
||||
},
|
||||
"bytes": 1905681
|
||||
"bytes": 1912658
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,214 @@
|
|||
import { log } from '../log.js';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
// import * as helpers from './helpers.js';
|
||||
// import * as profile from '../profile.js';
|
||||
|
||||
const models = {};
|
||||
let config = {};
|
||||
const anchors = [];
|
||||
const kMidHipCenter = 0;
|
||||
const kPoseDetectKeyNum = 2;
|
||||
const kFullBodySizeRot = 1;
|
||||
|
||||
function calculateScale(min_scale, max_scale, stride_index, num_strides) {
|
||||
if (num_strides === 1) return (min_scale + max_scale) * 0.5;
|
||||
return min_scale + (max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0);
|
||||
}
|
||||
|
||||
export function generateAnchors() {
|
||||
const options = {};
|
||||
options.strides = [];
|
||||
options.aspect_ratios = [];
|
||||
options.feature_map_height = [];
|
||||
options.num_layers = 4;
|
||||
options.min_scale = 0.1484375;
|
||||
options.max_scale = 0.75;
|
||||
options.input_size_height = 128;
|
||||
options.input_size_width = 128;
|
||||
options.anchor_offset_x = 0.5;
|
||||
options.anchor_offset_y = 0.5;
|
||||
options.strides.push(8);
|
||||
options.strides.push(16);
|
||||
options.strides.push(16);
|
||||
options.strides.push(16);
|
||||
options.aspect_ratios.push(1.0);
|
||||
options.reduce_boxes_in_lowest_layer = false;
|
||||
options.interpolated_scale_aspect_ratio = 1.0;
|
||||
options.fixed_anchor_size = true;
|
||||
let layer_id = 0;
|
||||
while (layer_id < options.strides.length) {
|
||||
const anchor_height = [];
|
||||
const anchor_width = [];
|
||||
const aspect_ratios = [];
|
||||
const scales = [];
|
||||
// For same strides, we merge the anchors in the same order.
|
||||
let last_same_stride_layer = layer_id;
|
||||
while (last_same_stride_layer < options.strides.length && options.strides[last_same_stride_layer] === options.strides[layer_id]) {
|
||||
const scale = calculateScale(options.min_scale, options.max_scale, last_same_stride_layer, options.strides.length);
|
||||
if (last_same_stride_layer === 0 && options.reduce_boxes_in_lowest_layer) {
|
||||
// For first layer, it can be specified to use predefined anchors.
|
||||
aspect_ratios.push(1.0);
|
||||
aspect_ratios.push(2.0);
|
||||
aspect_ratios.push(0.5);
|
||||
scales.push(0.1);
|
||||
scales.push(scale);
|
||||
scales.push(scale);
|
||||
} else {
|
||||
for (let aspect_ratio_id = 0; aspect_ratio_id < options.aspect_ratios.length; ++aspect_ratio_id) {
|
||||
aspect_ratios.push(options.aspect_ratios[aspect_ratio_id]);
|
||||
scales.push(scale);
|
||||
}
|
||||
if (options.interpolated_scale_aspect_ratio > 0.0) {
|
||||
const scale_next = last_same_stride_layer === options.strides.length - 1 ? 1.0 : calculateScale(options.min_scale, options.max_scale, last_same_stride_layer + 1, options.strides.length);
|
||||
scales.push(Math.sqrt(scale * scale_next));
|
||||
aspect_ratios.push(options.interpolated_scale_aspect_ratio);
|
||||
}
|
||||
}
|
||||
last_same_stride_layer++;
|
||||
}
|
||||
for (let i = 0; i < aspect_ratios.length; ++i) {
|
||||
const ratio_sqrts = Math.sqrt(aspect_ratios[i]);
|
||||
anchor_height.push(scales[i] / ratio_sqrts);
|
||||
anchor_width.push(scales[i] * ratio_sqrts);
|
||||
}
|
||||
let feature_map_height = 0;
|
||||
let feature_map_width = 0;
|
||||
if (options.feature_map_height.length) {
|
||||
feature_map_height = options.feature_map_height[layer_id];
|
||||
feature_map_width = options.feature_map_width[layer_id];
|
||||
} else {
|
||||
const stride = options.strides[layer_id];
|
||||
feature_map_height = Math.ceil(1.0 * options.input_size_height / stride);
|
||||
feature_map_width = Math.ceil(1.0 * options.input_size_width / stride);
|
||||
}
|
||||
for (let y = 0; y < feature_map_height; ++y) {
|
||||
for (let x = 0; x < feature_map_width; ++x) {
|
||||
for (let anchor_id = 0; anchor_id < anchor_height.length; ++anchor_id) {
|
||||
const x_center = (x + options.anchor_offset_x) * 1.0 / feature_map_width;
|
||||
const y_center = (y + options.anchor_offset_y) * 1.0 / feature_map_height;
|
||||
const new_anchor = {};
|
||||
new_anchor.x_center = x_center;
|
||||
new_anchor.y_center = y_center;
|
||||
if (options.fixed_anchor_size) {
|
||||
new_anchor.w = 1.0;
|
||||
new_anchor.h = 1.0;
|
||||
} else {
|
||||
new_anchor.w = anchor_width[anchor_id];
|
||||
new_anchor.h = anchor_height[anchor_id];
|
||||
}
|
||||
anchors.push(new_anchor);
|
||||
}
|
||||
}
|
||||
}
|
||||
layer_id = last_same_stride_layer;
|
||||
}
|
||||
}
|
||||
|
||||
export async function load(cfg) {
|
||||
config = cfg;
|
||||
if (!models.blazepose) {
|
||||
models.blazepose = await tf.loadGraphModel(config.pose.modelPath);
|
||||
log(`load model: ${config.pose.modelPath.match(/\/(.*)\./)[1]}`);
|
||||
}
|
||||
generateAnchors();
|
||||
return models.blazepose;
|
||||
}
|
||||
|
||||
function rotateRegion(region) {
|
||||
const x0 = region.keys[kMidHipCenter].x;
|
||||
const y0 = region.keys[kMidHipCenter].y;
|
||||
const x1 = (region.box[0] + region.box[2]) * 0.5;
|
||||
const y1 = (region.box[1] + region.box[3]) * 0.5;
|
||||
const target_angle = Math.PI * 0.5;
|
||||
const angle = target_angle - Math.atan2(-(y1 - y0), x1 - x0);
|
||||
return Math.round(1000 * (angle - 2 * Math.PI * Math.floor((angle - (-Math.PI)) / (2 * Math.PI)))) / 1000;
|
||||
}
|
||||
|
||||
function rotateVecor(vec, rotation) {
|
||||
const sx = vec.x;
|
||||
const sy = vec.y;
|
||||
vec.x = sx * Math.cos(rotation) - sy * Math.sin(rotation);
|
||||
vec.y = sx * Math.sin(rotation) + sy * Math.cos(rotation);
|
||||
}
|
||||
|
||||
async function decode(logits) {
|
||||
const scores = await logits[0].data();
|
||||
const boxes = await logits[1].data();
|
||||
// todo: add nms
|
||||
// todo scale output with image.shape
|
||||
const regions = [];
|
||||
for (let i = 0; i < anchors.length; i++) {
|
||||
const region = {};
|
||||
const score = 1.0 / (1.0 + Math.exp(-scores[i]));
|
||||
if (score > config.pose.scoreThreshold) {
|
||||
const idx = (4 + 2 * kPoseDetectKeyNum) * i;
|
||||
/* boundary box */
|
||||
const sx = boxes[idx + 0];
|
||||
const sy = boxes[idx + 1];
|
||||
const w = boxes[idx + 2] / config.pose.inputSize;
|
||||
const h = boxes[idx + 3] / config.pose.inputSize;
|
||||
const cx = (sx + anchors[i].x_center * config.pose.inputSize) / config.pose.inputSize;
|
||||
const cy = (sy + anchors[i].y_center * config.pose.inputSize) / config.pose.inputSize;
|
||||
region.score = Math.round(1000 * score) / 1000;
|
||||
region.box = [cx - w * 0.5, cy - h * 0.5, w * 0.5, h * 0.5];
|
||||
/* landmark positions (6 keys) */
|
||||
const keys = new Array(kPoseDetectKeyNum);
|
||||
for (let j = 0; j < kPoseDetectKeyNum; j++) {
|
||||
const lx = (boxes[idx + 4 + (2 * j) + 0] + anchors[i].x_center * config.pose.inputSize) / config.pose.inputSize;
|
||||
const ly = (boxes[idx + 4 + (2 * j) + 1] + anchors[i].y_center * config.pose.inputSize) / config.pose.inputSize;
|
||||
keys[j] = { x: lx, y: ly };
|
||||
}
|
||||
region.keys = keys;
|
||||
region.angle = rotateRegion(region);
|
||||
// add points
|
||||
const x_center = region.keys[kMidHipCenter].x * config.pose.inputSize;
|
||||
const y_center = region.keys[kMidHipCenter].y * config.pose.inputSize;
|
||||
const x_scale = region.keys[kFullBodySizeRot].x * config.pose.inputSize;
|
||||
const y_scale = region.keys[kFullBodySizeRot].y * config.pose.inputSize;
|
||||
// Bounding box size as double distance from center to scale point.
|
||||
const box_size = Math.sqrt((x_scale - x_center) * (x_scale - x_center) + (y_scale - y_center) * (y_scale - y_center)) * 2.0;
|
||||
/* RectTransformationCalculator::TransformNormalizedRect() */
|
||||
const roi_cx = region.angle === 0.0 ? x_center + box_size : x_center + box_size * Math.cos(region.angle) - box_size * Math.sin(region.angle);
|
||||
const roi_cy = region.angle === 0.0 ? y_center + box_size : y_center + box_size * Math.sin(region.angle) + box_size * Math.cos(region.angle);
|
||||
const long_side = Math.max(box_size, box_size);
|
||||
const roi_w = long_side * 1.5;
|
||||
const roi_h = long_side * 1.5;
|
||||
region.center = { x: roi_cx / config.pose.inputSize, y: roi_cy / config.pose.inputSize };
|
||||
region.size = { x: roi_w / config.pose.inputSize, y: roi_h / config.pose.inputSize };
|
||||
/* calculate ROI coordinates */
|
||||
const dx = roi_w * 0.5;
|
||||
const dy = roi_h * 0.5;
|
||||
region.coords = [];
|
||||
region.coords[0] = { x: -dx, y: -dy };
|
||||
region.coords[1] = { x: +dx, y: -dy };
|
||||
region.coords[2] = { x: +dx, y: +dy };
|
||||
region.coords[3] = { x: -dx, y: +dy };
|
||||
for (let j = 0; j < 4; j++) {
|
||||
rotateVecor(region.coords[i], region.angle);
|
||||
region.coords[j].x = (region.coords[j].x + roi_cx) / config.pose.inputSize;
|
||||
region.coords[j].y = (region.coords[j].y + roi_cy) / config.pose.inputSize;
|
||||
}
|
||||
|
||||
regions.push(region);
|
||||
}
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
|
||||
export async function predict(image, cfg) {
|
||||
if (!models.blazepose) return null;
|
||||
return new Promise(async (resolve) => {
|
||||
config = cfg;
|
||||
const resize = tf.image.resizeBilinear(image, [config.pose.inputSize, config.pose.inputSize], false);
|
||||
const enhance = tf.div(resize, 127.5).sub(1);
|
||||
tf.dispose(resize);
|
||||
const logits = await models.blazepose.predict(enhance);
|
||||
// todo: add landmarks model
|
||||
tf.dispose(enhance);
|
||||
const regions = await decode(logits);
|
||||
logits[0].dispose();
|
||||
logits[1].dispose();
|
||||
log('poses', regions);
|
||||
resolve(regions);
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,295 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as mp from './helpers.js';
|
||||
|
||||
/* ------------------------------------------------ *
|
||||
* The MIT License (MIT)
|
||||
* Copyright (c) 2020 terryky1220@gmail.com
|
||||
* ------------------------------------------------ */
|
||||
|
||||
const kMidHipCenter = 0;
|
||||
const kFullBodySizeRot = 1;
|
||||
const kMidShoulderCenter = 2;
|
||||
const kUpperBodySizeRot = 3;
|
||||
const kPoseDetectKeyNum = 2;
|
||||
const POSE_JOINT_NUM = 33;
|
||||
let s_detect_model;
|
||||
let s_detect_tensor_input;
|
||||
let s_landmark_model;
|
||||
let s_landmark_tensor_input;
|
||||
const s_anchors = [];
|
||||
|
||||
function create_ssd_anchors() {
|
||||
const anchor_options = {};
|
||||
anchor_options.strides = [];
|
||||
anchor_options.aspect_ratios = [];
|
||||
anchor_options.feature_map_height = [];
|
||||
anchor_options.num_layers = 4;
|
||||
anchor_options.min_scale = 0.1484375;
|
||||
anchor_options.max_scale = 0.75;
|
||||
anchor_options.input_size_height = 128;
|
||||
anchor_options.input_size_width = 128;
|
||||
anchor_options.anchor_offset_x = 0.5;
|
||||
anchor_options.anchor_offset_y = 0.5;
|
||||
anchor_options.strides.push(8);
|
||||
anchor_options.strides.push(16);
|
||||
anchor_options.strides.push(16);
|
||||
anchor_options.strides.push(16);
|
||||
anchor_options.aspect_ratios.push(1.0);
|
||||
anchor_options.reduce_boxes_in_lowest_layer = false;
|
||||
anchor_options.interpolated_scale_aspect_ratio = 1.0;
|
||||
anchor_options.fixed_anchor_size = true;
|
||||
mp.GenerateAnchors(s_anchors, anchor_options);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* Create TensorFlow.js Model
|
||||
* -------------------------------------------------- */
|
||||
async function init_tfjs_blazepose() {
|
||||
const url = './model/tfjs_model_full_pose_detection_float32/model.json';
|
||||
s_detect_model = await tf.loadGraphModel(url);
|
||||
const url_landmark = './model/tfjs_model_full_pose_landmark_39kp_float32/model.json';
|
||||
s_landmark_model = await tf.loadGraphModel(url_landmark);
|
||||
/* Pose detect */
|
||||
s_detect_tensor_input = tfjs_get_tensor_by_name(s_detect_model, 0, 'input');
|
||||
/* Pose Landmark */
|
||||
s_landmark_tensor_input = tfjs_get_tensor_by_name(s_landmark_model, 0, 'input');
|
||||
const det_input_w = s_detect_tensor_input.shape[2];
|
||||
const det_input_h = s_detect_tensor_input.shape[1];
|
||||
create_ssd_anchors(det_input_w, det_input_h);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* Invoke TensorFlow.js (Pose detection)
|
||||
* -------------------------------------------------- */
|
||||
async function decode_bounds(region_list, logits, score_thresh, input_img_w, input_img_h) {
|
||||
const scores_ptr = await logits[0].data();
|
||||
const bbox_ptr = await logits[1].data();
|
||||
for (let i = 0; i < s_anchors.length; i++) {
|
||||
const region = {};
|
||||
const anchor = s_anchors[i];
|
||||
const score0 = scores_ptr[i];
|
||||
const score = 1.0 / (1.0 + Math.exp(-score0));
|
||||
if (score > score_thresh) {
|
||||
/*
|
||||
* cx, cy, width, height
|
||||
* key0_x, key0_y
|
||||
* key1_x, key1_y
|
||||
*/
|
||||
const numkey = kPoseDetectKeyNum;
|
||||
const bbx_idx = (4 + 2 * numkey) * i;
|
||||
/* boundary box */
|
||||
const sx = bbox_ptr[bbx_idx + 0];
|
||||
const sy = bbox_ptr[bbx_idx + 1];
|
||||
let w = bbox_ptr[bbx_idx + 2];
|
||||
let h = bbox_ptr[bbx_idx + 3];
|
||||
let cx = sx + anchor.x_center * input_img_w;
|
||||
let cy = sy + anchor.y_center * input_img_h;
|
||||
cx /= input_img_w;
|
||||
cy /= input_img_h;
|
||||
w /= input_img_w;
|
||||
h /= input_img_h;
|
||||
const topleft = {};
|
||||
const btmright = {};
|
||||
topleft.x = cx - w * 0.5;
|
||||
topleft.y = cy - h * 0.5;
|
||||
btmright.x = cx + w * 0.5;
|
||||
btmright.y = cy + h * 0.5;
|
||||
region.score = score;
|
||||
region.topleft = topleft;
|
||||
region.btmright = btmright;
|
||||
/* landmark positions (6 keys) */
|
||||
const keys = new Array(kPoseDetectKeyNum);
|
||||
for (let j = 0; j < kPoseDetectKeyNum; j++) {
|
||||
let lx = bbox_ptr[bbx_idx + 4 + (2 * j) + 0];
|
||||
let ly = bbox_ptr[bbx_idx + 4 + (2 * j) + 1];
|
||||
lx += anchor.x_center * input_img_w;
|
||||
ly += anchor.y_center * input_img_h;
|
||||
lx /= input_img_w;
|
||||
ly /= input_img_h;
|
||||
keys[j] = { x: lx, y: ly };
|
||||
}
|
||||
region.keys = keys;
|
||||
region_list.push(region);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* extract ROI
|
||||
* based on:
|
||||
* - mediapipe/calculators/util/alignment_points_to_rects_calculator.cc
|
||||
* AlignmentPointsRectsCalculator::DetectionToNormalizedRect()
|
||||
* - mediapipe\calculators\util\rect_transformation_calculator.cc
|
||||
* RectTransformationCalculator::TransformNormalizedRect()
|
||||
* -------------------------------------------------- */
|
||||
function normalize_radians(angle) {
|
||||
return angle - 2 * Math.PI * Math.floor((angle - (-Math.PI)) / (2 * Math.PI));
|
||||
}
|
||||
|
||||
function compute_rotation(region) {
|
||||
const x0 = region.keys[kMidHipCenter].x;
|
||||
const y0 = region.keys[kMidHipCenter].y;
|
||||
const x1 = (region.topleft.x + region.btmright.x) * 0.5;
|
||||
const y1 = (region.topleft.y + region.btmright.y) * 0.5;
|
||||
const target_angle = Math.PI * 0.5;
|
||||
const rotation = target_angle - Math.atan2(-(y1 - y0), x1 - x0);
|
||||
region.rotation = normalize_radians(rotation);
|
||||
}
|
||||
|
||||
function rot_vec(vec, rotation) {
|
||||
const sx = vec.x;
|
||||
const sy = vec.y;
|
||||
vec.x = sx * Math.cos(rotation) - sy * Math.sin(rotation);
|
||||
vec.y = sx * Math.sin(rotation) + sy * Math.cos(rotation);
|
||||
}
|
||||
|
||||
function compute_detect_to_roi(region) {
|
||||
const input_img_w = s_detect_tensor_input.shape[2];
|
||||
const input_img_h = s_detect_tensor_input.shape[1];
|
||||
const x_center = region.keys[kMidHipCenter].x * input_img_w;
|
||||
const y_center = region.keys[kMidHipCenter].y * input_img_h;
|
||||
const x_scale = region.keys[kFullBodySizeRot].x * input_img_w;
|
||||
const y_scale = region.keys[kFullBodySizeRot].y * input_img_h;
|
||||
// Bounding box size as double distance from center to scale point.
|
||||
const box_size = Math.sqrt((x_scale - x_center) * (x_scale - x_center) + (y_scale - y_center) * (y_scale - y_center)) * 2.0;
|
||||
/* RectTransformationCalculator::TransformNormalizedRect() */
|
||||
const width = box_size;
|
||||
const height = box_size;
|
||||
const rotation = region.rotation;
|
||||
const shift_x = 0.0;
|
||||
const shift_y = 0.0;
|
||||
let roi_cx;
|
||||
let roi_cy;
|
||||
if (rotation === 0.0) {
|
||||
roi_cx = x_center + (width * shift_x);
|
||||
roi_cy = y_center + (height * shift_y);
|
||||
} else {
|
||||
const dx = (width * shift_x) * Math.cos(rotation) - (height * shift_y) * Math.sin(rotation);
|
||||
const dy = (width * shift_x) * Math.sin(rotation) + (height * shift_y) * Math.cos(rotation);
|
||||
roi_cx = x_center + dx;
|
||||
roi_cy = y_center + dy;
|
||||
}
|
||||
/*
|
||||
* calculate ROI width and height.
|
||||
* scale parameter is based on
|
||||
* "mediapipe/modules/pose_landmark/pose_detection_to_roi.pbtxt"
|
||||
*/
|
||||
const scale_x = 1.5;
|
||||
const scale_y = 1.5;
|
||||
const long_side = Math.max(width, height);
|
||||
const roi_w = long_side * scale_x;
|
||||
const roi_h = long_side * scale_y;
|
||||
region.roi_center = { x: roi_cx / input_img_w, y: roi_cy / input_img_h };
|
||||
region.roi_size = { x: roi_w / input_img_w, y: roi_h / input_img_h };
|
||||
/* calculate ROI coordinates */
|
||||
const dx = roi_w * 0.5;
|
||||
const dy = roi_h * 0.5;
|
||||
region.roi_coord = [];
|
||||
region.roi_coord[0] = { x: -dx, y: -dy };
|
||||
region.roi_coord[1] = { x: +dx, y: -dy };
|
||||
region.roi_coord[2] = { x: +dx, y: +dy };
|
||||
region.roi_coord[3] = { x: -dx, y: +dy };
|
||||
for (let i = 0; i < 4; i++) {
|
||||
rot_vec(region.roi_coord[i], rotation);
|
||||
region.roi_coord[i].x += roi_cx;
|
||||
region.roi_coord[i].y += roi_cy;
|
||||
region.roi_coord[i].x /= input_img_h;
|
||||
region.roi_coord[i].y /= input_img_h;
|
||||
}
|
||||
}
|
||||
|
||||
function pack_detect_result(detect_result, region_list) {
|
||||
for (let i = 0; i < region_list.length; i++) {
|
||||
region = region_list[i];
|
||||
compute_rotation(region);
|
||||
compute_detect_to_roi(region);
|
||||
detect_result.push(region);
|
||||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* Invoke TensorFlow.js (Pose detection)
|
||||
* -------------------------------------------------- */
|
||||
function exec_tfjs(img) {
|
||||
const w = s_detect_tensor_input.shape[2];
|
||||
const h = s_detect_tensor_input.shape[1];
|
||||
const logits = tf.tidy(() => {
|
||||
img_tensor1d = tf.tensor1d(img);
|
||||
img_tensor = img_tensor1d.reshape([h, w, 3]);
|
||||
// normalize [0, 255] to [-1, 1].
|
||||
const min = -1;
|
||||
const max = 1;
|
||||
const normalized = img_tensor.toFloat().mul((max - min) / 255.0).add(min);
|
||||
// resize, reshape
|
||||
const batched = normalized.reshape([-1, w, h, 3]);
|
||||
return s_detect_model.predict(batched);
|
||||
});
|
||||
return logits;
|
||||
}
|
||||
|
||||
async function invoke_pose_detect(img) {
|
||||
const logits = exec_tfjs(img);
|
||||
const score_thresh = 0.75;
|
||||
const detect_result = [];
|
||||
const region_list = [];
|
||||
const w = s_detect_tensor_input.shape[2];
|
||||
const h = s_detect_tensor_input.shape[1];
|
||||
await decode_bounds(region_list, logits, score_thresh, w, h);
|
||||
if (true) { /* USE NMS */
|
||||
const iou_thresh = 0.3;
|
||||
const region_nms_list = [];
|
||||
non_max_suppression(region_list, region_nms_list, iou_thresh);
|
||||
pack_detect_result(detect_result, region_nms_list);
|
||||
} else {
|
||||
pack_detect_result(detect_result, region_list);
|
||||
}
|
||||
/* release the resource of output tensor */
|
||||
logits[0].dispose();
|
||||
logits[1].dispose();
|
||||
return detect_result;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* Invoke TensorFlow.js (Pose landmark)
|
||||
* -------------------------------------------------- */
|
||||
function exec_tfjs_landmark(img) {
|
||||
const w = s_landmark_tensor_input.shape[2];
|
||||
const h = s_landmark_tensor_input.shape[1];
|
||||
const logits = tf.tidy(() => {
|
||||
img_tensor1d = tf.tensor1d(img);
|
||||
img_tensor = img_tensor1d.reshape([h, w, 3]);
|
||||
// normalize [0, 255] to [-1, 1].
|
||||
const min = -1;
|
||||
const max = 1;
|
||||
const normalized = img_tensor.toFloat().mul((max - min) / 255.0).add(min);
|
||||
// resize, reshape
|
||||
const batched = normalized.reshape([-1, w, h, 3]);
|
||||
return s_landmark_model.predict(batched);
|
||||
});
|
||||
return logits;
|
||||
}
|
||||
|
||||
async function invoke_pose_landmark(img) {
|
||||
const logits = exec_tfjs_landmark(img);
|
||||
const poseflag_ptr = await logits[1].data(); /* shape: (1, 1, 1, 1) */
|
||||
const landmark_ptr = await logits[2].data(); /* shape: (1, 156) */
|
||||
const img_w = s_landmark_tensor_input.shape[2];
|
||||
const img_h = s_landmark_tensor_input.shape[1];
|
||||
const landmark_result = {};
|
||||
landmark_result.joint = [];
|
||||
landmark_result.score = poseflag_ptr[0];
|
||||
for (let i = 0; i < POSE_JOINT_NUM; i++) {
|
||||
landmark_result.joint[i] = {
|
||||
x: landmark_ptr[4 * i + 0] / img_w,
|
||||
y: landmark_ptr[4 * i + 1] / img_h,
|
||||
z: landmark_ptr[4 * i + 2],
|
||||
};
|
||||
}
|
||||
logits[0].dispose();
|
||||
logits[1].dispose();
|
||||
logits[2].dispose();
|
||||
return landmark_result;
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
/* ------------------------------------------------ *
|
||||
* The MIT License (MIT)
|
||||
* Copyright (c) 2020 terryky1220@gmail.com
|
||||
* ------------------------------------------------ */
|
||||
|
||||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
function CalculateScale(min_scale, max_scale, stride_index, num_strides) {
|
||||
if (num_strides === 1) return (min_scale + max_scale) * 0.5;
|
||||
return min_scale + (max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0);
|
||||
}
|
||||
|
||||
export function GenerateAnchors(anchors, options) {
|
||||
let layer_id = 0;
|
||||
while (layer_id < options.strides.length) {
|
||||
const anchor_height = [];
|
||||
const anchor_width = [];
|
||||
const aspect_ratios = [];
|
||||
const scales = [];
|
||||
// For same strides, we merge the anchors in the same order.
|
||||
let last_same_stride_layer = layer_id;
|
||||
while (last_same_stride_layer < options.strides.length
|
||||
&& options.strides[last_same_stride_layer] === options.strides[layer_id]) {
|
||||
const scale = CalculateScale(options.min_scale, options.max_scale,
|
||||
last_same_stride_layer, options.strides.length);
|
||||
if (last_same_stride_layer === 0 && options.reduce_boxes_in_lowest_layer) {
|
||||
// For first layer, it can be specified to use predefined anchors.
|
||||
aspect_ratios.push(1.0);
|
||||
aspect_ratios.push(2.0);
|
||||
aspect_ratios.push(0.5);
|
||||
scales.push(0.1);
|
||||
scales.push(scale);
|
||||
scales.push(scale);
|
||||
} else {
|
||||
for (let aspect_ratio_id = 0;
|
||||
aspect_ratio_id < options.aspect_ratios.length;
|
||||
++aspect_ratio_id) {
|
||||
aspect_ratios.push(options.aspect_ratios[aspect_ratio_id]);
|
||||
scales.push(scale);
|
||||
}
|
||||
if (options.interpolated_scale_aspect_ratio > 0.0) {
|
||||
const scale_next = last_same_stride_layer === options.strides.length - 1
|
||||
? 1.0
|
||||
: CalculateScale(options.min_scale, options.max_scale,
|
||||
last_same_stride_layer + 1,
|
||||
options.strides.length);
|
||||
scales.push(Math.sqrt(scale * scale_next));
|
||||
aspect_ratios.push(options.interpolated_scale_aspect_ratio);
|
||||
}
|
||||
}
|
||||
last_same_stride_layer++;
|
||||
}
|
||||
for (let i = 0; i < aspect_ratios.length; ++i) {
|
||||
const ratio_sqrts = Math.sqrt(aspect_ratios[i]);
|
||||
anchor_height.push(scales[i] / ratio_sqrts);
|
||||
anchor_width.push(scales[i] * ratio_sqrts);
|
||||
}
|
||||
let feature_map_height = 0;
|
||||
let feature_map_width = 0;
|
||||
if (options.feature_map_height.length) {
|
||||
feature_map_height = options.feature_map_height[layer_id];
|
||||
feature_map_width = options.feature_map_width[layer_id];
|
||||
} else {
|
||||
const stride = options.strides[layer_id];
|
||||
feature_map_height = Math.ceil(1.0 * options.input_size_height / stride);
|
||||
feature_map_width = Math.ceil(1.0 * options.input_size_width / stride);
|
||||
}
|
||||
for (let y = 0; y < feature_map_height; ++y) {
|
||||
for (let x = 0; x < feature_map_width; ++x) {
|
||||
for (let anchor_id = 0; anchor_id < anchor_height.length; ++anchor_id) {
|
||||
// TODO: Support specifying anchor_offset_x, anchor_offset_y.
|
||||
const x_center = (x + options.anchor_offset_x) * 1.0 / feature_map_width;
|
||||
const y_center = (y + options.anchor_offset_y) * 1.0 / feature_map_height;
|
||||
const new_anchor = {};
|
||||
new_anchor.x_center = x_center;
|
||||
new_anchor.y_center = y_center;
|
||||
if (options.fixed_anchor_size) {
|
||||
new_anchor.w = 1.0;
|
||||
new_anchor.h = 1.0;
|
||||
} else {
|
||||
new_anchor.w = anchor_width[anchor_id];
|
||||
new_anchor.h = anchor_height[anchor_id];
|
||||
}
|
||||
anchors.push(new_anchor);
|
||||
}
|
||||
}
|
||||
}
|
||||
layer_id = last_same_stride_layer;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------- *
|
||||
* Apply NonMaxSuppression:
|
||||
* https://github.com/tensorflow/tfjs/blob/master/tfjs-core/src/ops/image_ops.ts
|
||||
* -------------------------------------------------- */
|
||||
function calc_intersection_over_union(region0, region1) {
|
||||
const sx0 = region0.box[0];
|
||||
const sy0 = region0.box[1];
|
||||
const ex0 = region0.box[2];
|
||||
const ey0 = region0.box[3];
|
||||
const sx1 = region1.box[0];
|
||||
const sy1 = region1.box[1];
|
||||
const ex1 = region1.box[2];
|
||||
const ey1 = region1.box[3];
|
||||
const xmin0 = Math.min(sx0, ex0);
|
||||
const ymin0 = Math.min(sy0, ey0);
|
||||
const xmax0 = Math.max(sx0, ex0);
|
||||
const ymax0 = Math.max(sy0, ey0);
|
||||
const xmin1 = Math.min(sx1, ex1);
|
||||
const ymin1 = Math.min(sy1, ey1);
|
||||
const xmax1 = Math.max(sx1, ex1);
|
||||
const ymax1 = Math.max(sy1, ey1);
|
||||
const area0 = (ymax0 - ymin0) * (xmax0 - xmin0);
|
||||
const area1 = (ymax1 - ymin1) * (xmax1 - xmin1);
|
||||
if (area0 <= 0 || area1 <= 0) return 0.0;
|
||||
const intersect_xmin = Math.max(xmin0, xmin1);
|
||||
const intersect_ymin = Math.max(ymin0, ymin1);
|
||||
const intersect_xmax = Math.min(xmax0, xmax1);
|
||||
const intersect_ymax = Math.min(ymax0, ymax1);
|
||||
const intersect_area = Math.max(intersect_ymax - intersect_ymin, 0.0) * Math.max(intersect_xmax - intersect_xmin, 0.0);
|
||||
return intersect_area / (area0 + area1 - intersect_area);
|
||||
}
|
||||
|
||||
function compare(v1, v2) {
|
||||
if (v1.score > v2.score) return 1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
export function NMS(region_list, region_nms_list, iou_thresh) {
|
||||
region_list.sort(compare);
|
||||
for (let i = 0; i < region_list.length; i++) {
|
||||
const region_candidate = region_list[i];
|
||||
let ignore_candidate = false;
|
||||
for (let j = 0; j < region_nms_list.length; j++) {
|
||||
const region_nms = region_nms_list[j];
|
||||
const iou = calc_intersection_over_union(region_candidate, region_nms);
|
||||
if (iou >= iou_thresh) {
|
||||
ignore_candidate = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!ignore_candidate) {
|
||||
region_nms_list.push(region_candidate);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,374 @@
|
|||
import * as detect from './detect.js';
|
||||
|
||||
/* ------------------------------------------------ *
|
||||
* The MIT License (MIT)
|
||||
* Copyright (c) 2020 terryky1220@gmail.com
|
||||
* ------------------------------------------------ */
|
||||
// tf.setBackend('wasm').then(() => startWebGL());
|
||||
|
||||
let s_debug_log;
|
||||
let s_rtarget_main;
|
||||
let s_rtarget_feed;
|
||||
let s_rtarget_src;
|
||||
|
||||
class GuiProperty {
|
||||
constructor() {
|
||||
this.draw_roi_rect = false;
|
||||
this.draw_pmeter = false;
|
||||
}
|
||||
}
|
||||
const s_gui_prop = new GuiProperty();
|
||||
|
||||
function generate_input_image(gl, texid, win_w, win_h) {
|
||||
const dims = detect.get_pose_detect_input_dims();
|
||||
const buf_rgba = new Uint8Array(dims.w * dims.h * 4);
|
||||
const buf_rgb = new Uint8Array(dims.w * dims.h * 3);
|
||||
GLUtil.set_render_target(gl, s_rtarget_feed);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
r2d.draw_2d_texture(gl, texid, 0, win_h - dims.h, dims.w, dims.h, 1);
|
||||
gl.readPixels(0, 0, dims.w, dims.h, gl.RGBA, gl.UNSIGNED_BYTE, buf_rgba);
|
||||
for (let i = 0, j = 0; i < buf_rgba.length; i++) {
|
||||
if (i % 4 !== 3) buf_rgb[j++] = buf_rgba[i];
|
||||
}
|
||||
GLUtil.set_render_target(gl, s_rtarget_main);
|
||||
return buf_rgb;
|
||||
}
|
||||
|
||||
function generate_landmark_input_image(gl, srctex, texw, texh, detection, pose_id) {
|
||||
const dims = get_pose_landmark_input_dims();
|
||||
const buf_rgba = new Uint8Array(dims.w * dims.h * 4);
|
||||
const buf_rgb = new Uint8Array(dims.w * dims.h * 3);
|
||||
const texcoord = [];
|
||||
if (detection.length > pose_id) {
|
||||
region = detection[pose_id];
|
||||
const x0 = region.roi_coord[0].x;
|
||||
const y0 = region.roi_coord[0].y;
|
||||
const x1 = region.roi_coord[1].x; // 0--------1
|
||||
const y1 = region.roi_coord[1].y; // | |
|
||||
const x2 = region.roi_coord[2].x; // | |
|
||||
const y2 = region.roi_coord[2].y; // 3--------2
|
||||
const x3 = region.roi_coord[3].x;
|
||||
const y3 = region.roi_coord[3].y;
|
||||
texcoord[0] = x3; texcoord[1] = y3;
|
||||
texcoord[2] = x0; texcoord[3] = y0;
|
||||
texcoord[4] = x2; texcoord[5] = y2;
|
||||
texcoord[6] = x1; texcoord[7] = y1;
|
||||
}
|
||||
GLUtil.set_render_target(gl, s_rtarget_feed);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
r2d.draw_2d_texture_texcoord_rot(gl, srctex, 0, texh - dims.h, dims.w, dims.h, texcoord, 0, 0, 0);
|
||||
gl.readPixels(0, 0, dims.w, dims.h, gl.RGBA, gl.UNSIGNED_BYTE, buf_rgba);
|
||||
for (let i = 0, j = 0; i < buf_rgba.length; i++) {
|
||||
if (i % 4 != 3) buf_rgb[j++] = buf_rgba[i];
|
||||
}
|
||||
GLUtil.set_render_target(gl, s_rtarget_main);
|
||||
return buf_rgb;
|
||||
}
|
||||
|
||||
function
|
||||
render_detect_region(gl, ofstx, ofsty, texw, texh, detection) {
|
||||
const col_white = [1.0, 1.0, 1.0, 1.0];
|
||||
const col_red = [1.0, 0.0, 0.0, 1.0];
|
||||
const col_frame = [1.0, 0.0, 0.0, 1.0];
|
||||
for (let i = 0; i < detection.length; i++) {
|
||||
region = detection[i];
|
||||
const x1 = region.topleft.x * texw + ofstx;
|
||||
const y1 = region.topleft.y * texh + ofsty;
|
||||
const x2 = region.btmright.x * texw + ofstx;
|
||||
const y2 = region.btmright.y * texh + ofsty;
|
||||
const score = region.score;
|
||||
/* rectangle region */
|
||||
r2d.draw_2d_rect(gl, x1, y1, x2 - x1, y2 - y1, col_frame, 2.0);
|
||||
/* class name */
|
||||
const buf = '' + (score * 100).toFixed(0);
|
||||
dbgstr.draw_dbgstr_ex(gl, buf, x1, y1, 1.0, col_white, col_frame);
|
||||
/* key points */
|
||||
const hx = region.keys[kMidHipCenter].x * texw + ofstx;
|
||||
const hy = region.keys[kMidHipCenter].y * texh + ofsty;
|
||||
let sx = (region.topleft.x + region.btmright.x) * 0.5;
|
||||
let sy = (region.topleft.y + region.btmright.y) * 0.5;
|
||||
sx = sx * texw + ofstx;
|
||||
sy = sy * texh + ofsty;
|
||||
r2d.draw_2d_line(gl, hx, hy, sx, sy, col_white, 2.0);
|
||||
let r = 4;
|
||||
r2d.draw_2d_fillrect(gl, hx - (r / 2), hy - (r / 2), r, r, col_frame);
|
||||
r2d.draw_2d_fillrect(gl, sx - (r / 2), sy - (r / 2), r, r, col_frame);
|
||||
for (let j0 = 0; j0 < 4; j0++) {
|
||||
const j1 = (j0 + 1) % 4;
|
||||
const x1 = region.roi_coord[j0].x * texw + ofstx;
|
||||
const y1 = region.roi_coord[j0].y * texh + ofsty;
|
||||
const x2 = region.roi_coord[j1].x * texw + ofstx;
|
||||
const y2 = region.roi_coord[j1].y * texh + ofsty;
|
||||
r2d.draw_2d_line(gl, x1, y1, x2, y2, col_red, 2.0);
|
||||
}
|
||||
const cx = region.roi_center.x * texw + ofstx;
|
||||
const cy = region.roi_center.y * texh + ofsty;
|
||||
r = 10;
|
||||
r2d.draw_2d_fillrect(gl, cx - (r / 2), cy - (r / 2), r, r, col_red);
|
||||
}
|
||||
}
|
||||
|
||||
function transform_pose_landmark(transformed_pos, landmark, region) {
|
||||
const scale_x = region.roi_size.x;
|
||||
const scale_y = region.roi_size.y;
|
||||
const pivot_x = region.roi_center.x;
|
||||
const pivot_y = region.roi_center.y;
|
||||
const rotation = region.rotation;
|
||||
const mat = new Array(16);
|
||||
matrix_identity(mat);
|
||||
matrix_translate(mat, pivot_x, pivot_y, 0);
|
||||
matrix_rotate(mat, RAD_TO_DEG(rotation), 0, 0, 1);
|
||||
matrix_scale(mat, scale_x, scale_y, 1.0);
|
||||
matrix_translate(mat, -0.5, -0.5, 0);
|
||||
for (let i = 0; i < POSE_JOINT_NUM; i++) {
|
||||
const vec = [landmark.joint[i].x, landmark.joint[i].y];
|
||||
matrix_multvec2(mat, vec, vec);
|
||||
transformed_pos[i] = { x: vec[0], y: vec[1] };
|
||||
}
|
||||
}
|
||||
|
||||
function render_bone(gl, ofstx, ofsty, drw_w, drw_h,
|
||||
transformed_pos, id0, id1, col) {
|
||||
const x0 = transformed_pos[id0].x * drw_w + ofstx;
|
||||
const y0 = transformed_pos[id0].y * drw_h + ofsty;
|
||||
const x1 = transformed_pos[id1].x * drw_w + ofstx;
|
||||
const y1 = transformed_pos[id1].y * drw_h + ofsty;
|
||||
r2d.draw_2d_line(gl, x0, y0, x1, y1, col, 5.0);
|
||||
}
|
||||
|
||||
function render_pose_landmark(gl, ofstx, ofsty, texw, texh, landmakr_ret,
|
||||
detection, pose_id) {
|
||||
const col_red = [1.0, 0.0, 0.0, 1.0];
|
||||
const col_orange = [1.0, 0.6, 0.0, 1.0];
|
||||
const col_cyan = [0.0, 1.0, 1.0, 1.0];
|
||||
const col_lime = [0.0, 1.0, 0.3, 1.0];
|
||||
const col_pink = [1.0, 0.0, 1.0, 1.0];
|
||||
const col_blue = [0.0, 0.5, 1.0, 1.0];
|
||||
const col_white = [1.0, 1.0, 1.0, 1.0];
|
||||
if (landmakr_ret.length <= pose_id) return;
|
||||
const landmark = landmakr_ret[pose_id];
|
||||
const score = landmark.score;
|
||||
const buf = 'score:' + (score * 100).toFixed(1);
|
||||
dbgstr.draw_dbgstr_ex(gl, buf, texw - 120, 0, 1.0, col_white, col_red);
|
||||
const transformed_pos = new Array(POSE_JOINT_NUM);
|
||||
transform_pose_landmark(transformed_pos, landmark, detection[pose_id]);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 11, 12, col_cyan);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 12, 24, col_cyan);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 24, 23, col_cyan);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 23, 11, col_cyan);
|
||||
/* right arm */
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 11, 13, col_orange);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 13, 15, col_orange);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 15, 21, col_orange);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 15, 19, col_orange);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 15, 17, col_orange);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 17, 19, col_orange);
|
||||
/* left arm */
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 12, 14, col_lime);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 14, 16, col_lime);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 16, 22, col_lime);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 16, 20, col_lime);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 16, 18, col_lime);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 18, 20, col_lime);
|
||||
/* face */
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 9, 10, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 0, 1, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 1, 2, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 2, 3, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 3, 7, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 0, 4, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 4, 5, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 5, 6, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 6, 8, col_blue);
|
||||
/* right leg */
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 23, 25, col_pink);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 25, 27, col_pink);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 27, 31, col_pink);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 31, 29, col_pink);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 29, 27, col_pink);
|
||||
/* left leg */
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 24, 26, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 26, 28, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 28, 32, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 32, 30, col_blue);
|
||||
render_bone(gl, ofstx, ofsty, texw, texh, transformed_pos, 30, 28, col_blue);
|
||||
for (let i = 0; i < POSE_JOINT_NUM; i++) {
|
||||
const x = transformed_pos[i].x * texw + ofstx;
|
||||
const y = transformed_pos[i].y * texh + ofsty;
|
||||
const r = 9;
|
||||
r2d.draw_2d_fillrect(gl, x - (r / 2), y - (r / 2), r, r, col_red);
|
||||
}
|
||||
}
|
||||
|
||||
function render_cropped_pose_image(gl, srctex, ofstx, ofsty, texw, texh, detection, pose_id) {
|
||||
const texcoord = [];
|
||||
if (detection.length <= pose_id) return;
|
||||
region = detection[pose_id];
|
||||
const x0 = region.roi_coord[0].x;
|
||||
const y0 = region.roi_coord[0].y;
|
||||
const x1 = region.roi_coord[1].x; // 0--------1
|
||||
const y1 = region.roi_coord[1].y; // | |
|
||||
const x2 = region.roi_coord[2].x; // | |
|
||||
const y2 = region.roi_coord[2].y; // 3--------2
|
||||
const x3 = region.roi_coord[3].x;
|
||||
const y3 = region.roi_coord[3].y;
|
||||
texcoord[0] = x0; texcoord[1] = y0;
|
||||
texcoord[2] = x3; texcoord[3] = y3;
|
||||
texcoord[4] = x1; texcoord[5] = y1;
|
||||
texcoord[6] = x2; texcoord[7] = y2;
|
||||
r2d.draw_2d_texture_texcoord_rot(gl, srctex, ofstx, ofsty, texw, texh, texcoord, 0, 0, 0);
|
||||
}
|
||||
|
||||
/* Adjust the texture size to fit the window size
|
||||
*
|
||||
* Portrait
|
||||
* Landscape +------+
|
||||
* +-+------+-+ +------+
|
||||
* | | | | | |
|
||||
* | | | | | |
|
||||
* +-+------+-+ +------+
|
||||
* +------+
|
||||
*/
|
||||
function
|
||||
generate_squared_src_image(gl, texid, src_w, src_h, win_w, win_h) {
|
||||
const win_aspect = win_w / win_h;
|
||||
const tex_aspect = src_w / src_h;
|
||||
let scale;
|
||||
let scaled_w;
|
||||
let scaled_h;
|
||||
let offset_x;
|
||||
let offset_y;
|
||||
if (win_aspect > tex_aspect) {
|
||||
scale = win_h / src_h;
|
||||
scaled_w = scale * src_w;
|
||||
scaled_h = scale * src_h;
|
||||
offset_x = (win_w - scaled_w) * 0.5;
|
||||
offset_y = 0;
|
||||
} else {
|
||||
scale = win_w / src_w;
|
||||
scaled_w = scale * src_w;
|
||||
scaled_h = scale * src_h;
|
||||
offset_x = 0;
|
||||
offset_y = (win_h - scaled_h) * 0.5;
|
||||
}
|
||||
GLUtil.set_render_target(gl, s_rtarget_src);
|
||||
gl.clearColor(0.7, 0.7, 0.7, 1.0);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
r2d.draw_2d_texture(gl, texid, offset_x, offset_y, scaled_w, scaled_h, 1);
|
||||
}
|
||||
|
||||
function init_gui() {
|
||||
const gui = new dat.GUI();
|
||||
gui.add(s_gui_prop, 'draw_roi_rect');
|
||||
gui.add(s_gui_prop, 'draw_pmeter');
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- *
|
||||
* M A I N F U N C T I O N
|
||||
* ---------------------------------------------------------------- */
|
||||
async function startWebGL() {
|
||||
s_debug_log = document.getElementById('debug_log');
|
||||
const canvas = document.querySelector('#glcanvas');
|
||||
const gl = canvas.getContext('webgl');
|
||||
if (!gl) {
|
||||
alert('Failed to initialize WebGL.');
|
||||
return;
|
||||
}
|
||||
gl.clearColor(0.7, 0.7, 0.7, 1.0);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
init_gui();
|
||||
const camtex = GLUtil.create_camera_texture(gl);
|
||||
// const camtex = GLUtil.create_video_texture (gl, "pexels_dance.mp4");
|
||||
const imgtex = GLUtil.create_image_texture2(gl, 'pexels.jpg');
|
||||
const win_w = canvas.clientWidth;
|
||||
const win_h = canvas.clientHeight;
|
||||
r2d.init_2d_render(gl, win_w, win_h);
|
||||
init_dbgstr(gl, win_w, win_h);
|
||||
pmeter.init_pmeter(gl, win_w, win_h, win_h - 40);
|
||||
const stats = init_stats();
|
||||
await init_tfjs_blazepose();
|
||||
s_debug_log.innerHTML = 'tfjs.Backend = ' + tf.getBackend() + '<br>';
|
||||
s_rtarget_main = GLUtil.create_render_target(gl, win_w, win_h, 0);
|
||||
s_rtarget_feed = GLUtil.create_render_target(gl, win_w, win_w, 1);
|
||||
s_rtarget_src = GLUtil.create_render_target(gl, win_w, win_w, 1);
|
||||
/* stop loading spinner */
|
||||
const spinner = document.getElementById('loading');
|
||||
spinner.classList.add('loaded');
|
||||
let prev_time_ms = performance.now();
|
||||
|
||||
async function render(now) {
|
||||
pmeter.reset_lap(0);
|
||||
pmeter.set_lap(0);
|
||||
const cur_time_ms = performance.now();
|
||||
const interval_ms = cur_time_ms - prev_time_ms;
|
||||
prev_time_ms = cur_time_ms;
|
||||
stats.begin();
|
||||
let src_w = imgtex.image.width;
|
||||
let src_h = imgtex.image.height;
|
||||
let texid = imgtex.texid;
|
||||
if (GLUtil.is_camera_ready(camtex)) {
|
||||
GLUtil.update_camera_texture(gl, camtex);
|
||||
src_w = camtex.video.videoWidth;
|
||||
src_h = camtex.video.videoHeight;
|
||||
texid = camtex.texid;
|
||||
}
|
||||
generate_squared_src_image(gl, texid, src_w, src_h, win_w, win_h);
|
||||
texid = s_rtarget_src.texid;
|
||||
/* --------------------------------------- *
|
||||
* invoke TF.js (Pose detection)
|
||||
* --------------------------------------- */
|
||||
const feed_image = generate_input_image(gl, texid, win_w, win_h);
|
||||
const time_invoke0_start = performance.now();
|
||||
const predictions = await invoke_pose_detect(feed_image);
|
||||
const time_invoke0 = performance.now() - time_invoke0_start;
|
||||
/* --------------------------------------- *
|
||||
* invoke TF.js (Pose landmark)
|
||||
* --------------------------------------- */
|
||||
const landmark_ret = [];
|
||||
let time_invoke1 = 0;
|
||||
for (let pose_id = 0; pose_id < predictions.length; pose_id++) {
|
||||
const feed_image = generate_landmark_input_image(gl, texid, win_w, win_h, predictions, pose_id);
|
||||
const time_invoke1_start = performance.now();
|
||||
landmark_ret[pose_id] = await invoke_pose_landmark(feed_image);
|
||||
time_invoke1 += performance.now() - time_invoke1_start;
|
||||
}
|
||||
|
||||
/* --------------------------------------- *
|
||||
* render scene
|
||||
* --------------------------------------- */
|
||||
GLUtil.set_render_target(gl, s_rtarget_main);
|
||||
gl.clear(gl.COLOR_BUFFER_BIT);
|
||||
r2d.draw_2d_texture(gl, texid, 0, 0, win_w, win_h, 0);
|
||||
if (s_gui_prop.draw_roi_rect) {
|
||||
render_detect_region(gl, 0, 0, win_w, win_h, predictions);
|
||||
/* draw cropped image of the pose area */
|
||||
for (let pose_id = 0; pose_id < predictions.length; pose_id++) {
|
||||
const w = 100;
|
||||
const h = 100;
|
||||
const x = win_w - w - 10;
|
||||
const y = h * pose_id + 20;
|
||||
const col_white = [1.0, 1.0, 1.0, 1.0];
|
||||
render_cropped_pose_image(gl, texid, x, y, w, h, predictions, pose_id);
|
||||
r2d.draw_2d_rect(gl, x, y, w, h, col_white, 2.0);
|
||||
}
|
||||
}
|
||||
for (let pose_id = 0; pose_id < predictions.length; pose_id++) {
|
||||
render_pose_landmark(gl, 0, 0, win_w, win_h, landmark_ret, predictions, pose_id);
|
||||
}
|
||||
/* --------------------------------------- *
|
||||
* post process
|
||||
* --------------------------------------- */
|
||||
if (s_gui_prop.draw_pmeter) {
|
||||
pmeter.draw_pmeter(gl, 0, 40);
|
||||
}
|
||||
let str = 'Interval: ' + interval_ms.toFixed(1) + ' [ms]';
|
||||
dbgstr.draw_dbgstr(gl, str, 10, 10);
|
||||
str = 'TF.js0 : ' + time_invoke0.toFixed(1) + ' [ms]';
|
||||
dbgstr.draw_dbgstr(gl, str, 10, 10 + 22 * 1);
|
||||
str = 'TF.js1 : ' + time_invoke1.toFixed(1) + ' [ms]';
|
||||
dbgstr.draw_dbgstr(gl, str, 10, 10 + 22 * 2);
|
||||
stats.end();
|
||||
requestAnimationFrame(render);
|
||||
}
|
||||
render();
|
||||
}
|
||||
62
src/human.js
62
src/human.js
|
|
@ -1,13 +1,14 @@
|
|||
import { log } from './log.js';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as backend from './tfjs/backend.js';
|
||||
import * as facemesh from './face/facemesh.js';
|
||||
import * as facemesh from './blazeface/facemesh.js';
|
||||
import * as age from './age/age.js';
|
||||
import * as gender from './gender/gender.js';
|
||||
import * as emotion from './emotion/emotion.js';
|
||||
import * as embedding from './embedding/embedding.js';
|
||||
import * as posenet from './body/posenet.js';
|
||||
import * as handpose from './hand/handpose.js';
|
||||
import * as posenet from './posenet/posenet.js';
|
||||
import * as blazepose from './blazepose/blazepose.js';
|
||||
import * as handpose from './handpose/handpose.js';
|
||||
import * as gesture from './gesture/gesture.js';
|
||||
import * as image from './image.js';
|
||||
import * as profile from './profile.js';
|
||||
|
|
@ -18,7 +19,6 @@ import * as app from '../package.json';
|
|||
// helper function: gets elapsed time on both browser and nodejs
|
||||
const now = () => {
|
||||
if (typeof performance !== 'undefined') return performance.now();
|
||||
// @ts-ignore
|
||||
return parseInt(Number(process.hrtime.bigint()) / 1000 / 1000);
|
||||
};
|
||||
|
||||
|
|
@ -73,7 +73,6 @@ class Human {
|
|||
}
|
||||
|
||||
profile() {
|
||||
// @ts-ignore
|
||||
if (this.config.profile) return profile.data;
|
||||
return {};
|
||||
}
|
||||
|
|
@ -104,7 +103,6 @@ class Human {
|
|||
}
|
||||
|
||||
simmilarity(embedding1, embedding2) {
|
||||
// @ts-ignore
|
||||
if (this.config.face.embedding.enabled) return embedding.simmilarity(embedding1, embedding2);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -134,37 +132,26 @@ class Human {
|
|||
this.models.embedding,
|
||||
this.models.posenet,
|
||||
this.models.handpose,
|
||||
this.models.blazepose,
|
||||
] = await Promise.all([
|
||||
// @ts-ignore
|
||||
this.models.facemesh || (this.config.face.enabled ? facemesh.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.age || ((this.config.face.enabled && this.config.face.age.enabled) ? age.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.gender || ((this.config.face.enabled && this.config.face.gender.enabled) ? gender.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.emotion || ((this.config.face.enabled && this.config.face.emotion.enabled) ? emotion.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.embedding || ((this.config.face.enabled && this.config.face.embedding.enabled) ? embedding.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.posenet || (this.config.body.enabled ? posenet.load(this.config) : null),
|
||||
// @ts-ignore
|
||||
this.models.handpose || (this.config.hand.enabled ? handpose.load(this.config) : null),
|
||||
this.models.blazepose || (this.config.pose.enabled ? blazepose.load(this.config) : null),
|
||||
]);
|
||||
} else {
|
||||
// @ts-ignore
|
||||
if (this.config.face.enabled && !this.models.facemesh) this.models.facemesh = await facemesh.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.face.enabled && this.config.face.age.enabled && !this.models.age) this.models.age = await age.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.face.enabled && this.config.face.gender.enabled && !this.models.gender) this.models.gender = await gender.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.face.enabled && this.config.face.emotion.enabled && !this.models.emotion) this.models.emotion = await emotion.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.face.enabled && this.config.face.embedding.enabled && !this.models.embedding) this.models.embedding = await embedding.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.body.enabled && !this.models.posenet) this.models.posenet = await posenet.load(this.config);
|
||||
// @ts-ignore
|
||||
if (this.config.hand.enabled && !this.models.handpose) this.models.handpose = await handpose.load(this.config);
|
||||
if (this.config.pose.enabled && !this.models.blazepose) this.models.blazepose = await blazepose.load(this.config);
|
||||
}
|
||||
const current = Math.trunc(now() - timeStamp);
|
||||
if (current > (this.perf.load || 0)) this.perf.load = current;
|
||||
|
|
@ -230,7 +217,6 @@ class Human {
|
|||
const faceRes = [];
|
||||
this.state = 'run:face';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
const faces = await this.models.facemesh?.estimateFaces(input, this.config);
|
||||
this.perf.face = Math.trunc(now() - timeStamp);
|
||||
for (const face of faces) {
|
||||
|
|
@ -245,12 +231,10 @@ class Human {
|
|||
// run age, inherits face from blazeface
|
||||
this.analyze('Start Age:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
ageRes = this.config.face.age.enabled ? age.predict(face.image, this.config) : {};
|
||||
} else {
|
||||
this.state = 'run:age';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
ageRes = this.config.face.age.enabled ? await age.predict(face.image, this.config) : {};
|
||||
this.perf.age = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
|
|
@ -258,12 +242,10 @@ class Human {
|
|||
// run gender, inherits face from blazeface
|
||||
this.analyze('Start Gender:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
genderRes = this.config.face.gender.enabled ? gender.predict(face.image, this.config) : {};
|
||||
} else {
|
||||
this.state = 'run:gender';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
genderRes = this.config.face.gender.enabled ? await gender.predict(face.image, this.config) : {};
|
||||
this.perf.gender = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
|
|
@ -271,12 +253,10 @@ class Human {
|
|||
// run emotion, inherits face from blazeface
|
||||
this.analyze('Start Emotion:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
emotionRes = this.config.face.emotion.enabled ? emotion.predict(face.image, this.config) : {};
|
||||
} else {
|
||||
this.state = 'run:emotion';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
emotionRes = this.config.face.emotion.enabled ? await emotion.predict(face.image, this.config) : {};
|
||||
this.perf.emotion = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
|
|
@ -285,12 +265,10 @@ class Human {
|
|||
// run emotion, inherits face from blazeface
|
||||
this.analyze('Start Embedding:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
embeddingRes = this.config.face.embedding.enabled ? embedding.predict(face.image, this.config) : {};
|
||||
} else {
|
||||
this.state = 'run:embedding';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
embeddingRes = this.config.face.embedding.enabled ? await embedding.predict(face.image, this.config) : {};
|
||||
this.perf.embedding = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
|
|
@ -346,7 +324,6 @@ class Human {
|
|||
async image(input, userConfig = {}) {
|
||||
this.state = 'image';
|
||||
this.config = mergeDeep(this.config, userConfig);
|
||||
// @ts-ignore
|
||||
const process = image.process(input, this.config);
|
||||
process.tensor.dispose();
|
||||
return process.canvas;
|
||||
|
|
@ -371,6 +348,7 @@ class Human {
|
|||
}
|
||||
|
||||
let poseRes;
|
||||
let blazeposeRes;
|
||||
let handRes;
|
||||
let faceRes;
|
||||
|
||||
|
|
@ -386,7 +364,6 @@ class Human {
|
|||
this.analyze('Start Scope:');
|
||||
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
const process = image.process(input, this.config);
|
||||
if (!process || !process.tensor) {
|
||||
log('could not convert input to tensor');
|
||||
|
|
@ -410,36 +387,45 @@ class Human {
|
|||
// run posenet
|
||||
this.analyze('Start Body:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
poseRes = this.config.body.enabled ? this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
if (this.perf.body) delete this.perf.body;
|
||||
} else {
|
||||
this.state = 'run:body';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
poseRes = this.config.body.enabled ? await this.models.posenet?.estimatePoses(process.tensor, this.config) : [];
|
||||
this.perf.body = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
this.analyze('End Body:');
|
||||
|
||||
// run posenet
|
||||
this.analyze('Start Pose:');
|
||||
if (this.config.async) {
|
||||
blazeposeRes = this.config.pose.enabled ? blazepose.predict(process.tensor, this.config) : [];
|
||||
if (this.perf.pose) delete this.perf.pose;
|
||||
} else {
|
||||
this.state = 'run:pose';
|
||||
timeStamp = now();
|
||||
blazeposeRes = this.config.pose.enabled ? await blazepose.predict(process.tensor, this.config) : [];
|
||||
this.perf.pose = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
this.analyze('End Pose:');
|
||||
|
||||
// run handpose
|
||||
this.analyze('Start Hand:');
|
||||
if (this.config.async) {
|
||||
// @ts-ignore
|
||||
handRes = this.config.hand.enabled ? this.models.handpose?.estimateHands(process.tensor, this.config) : [];
|
||||
if (this.perf.hand) delete this.perf.hand;
|
||||
} else {
|
||||
this.state = 'run:hand';
|
||||
timeStamp = now();
|
||||
// @ts-ignore
|
||||
handRes = this.config.hand.enabled ? await this.models.handpose?.estimateHands(process.tensor, this.config) : [];
|
||||
this.perf.hand = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
// this.analyze('End Hand:');
|
||||
this.analyze('End Hand:');
|
||||
|
||||
// if async wait for results
|
||||
if (this.config.async) {
|
||||
[faceRes, poseRes, handRes] = await Promise.all([faceRes, poseRes, handRes]);
|
||||
[faceRes, poseRes, blazeposeRes, handRes] = await Promise.all([faceRes, poseRes, blazeposeRes, handRes]);
|
||||
}
|
||||
process.tensor.dispose();
|
||||
|
||||
|
|
@ -456,7 +442,7 @@ class Human {
|
|||
|
||||
this.perf.total = Math.trunc(now() - timeStart);
|
||||
this.state = 'idle';
|
||||
resolve({ face: faceRes, body: poseRes, hand: handRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas });
|
||||
resolve({ face: faceRes, body: poseRes, hand: handRes, pose: blazeposeRes, gesture: gestureRes, performance: this.perf, canvas: process.canvas });
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue