ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

redo hand detection

pull/50/head
Vladimir Mandic 2020-11-04 01:11:24 -05:00
parent ced772bb43
commit b8c9687b29
27 changed files with 1416 additions and 1669 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

362
assets/tfjs-vis.min.js vendored
View File

File diff suppressed because one or more lines are too long

1
demo/index.html
View File

@ -16,7 +16,6 @@
<!-- <script src="../assets/tf.es2017.js"></script> --> <!-- <script src="../assets/tf.es2017.js"></script> -->
<!-- <script src="../assets/tf-backend-wasm.es2017.js"></script> --> <!-- <script src="../assets/tf-backend-wasm.es2017.js"></script> -->
<!-- <script src="../assets/tf-backend-webgpu.js"></script> --> <!-- <script src="../assets/tf-backend-webgpu.js"></script> -->
<!-- <script src='../assets/tfjs-vis.min.js'></script> -->
<script src="./browser.js" type="module"></script> <script src="./browser.js" type="module"></script>
<style> <style>
body { margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; overflow-x: hidden; scrollbar-width: none; } body { margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; overflow-x: hidden; scrollbar-width: none; }

32
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

100
dist/human.esm-nobundle.json vendored
View File

@ -25,7 +25,7 @@
"imports": [] "imports": []
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytes": 2568, "bytes": 2572,
"imports": [ "imports": [
{ {
"path": "src/facemesh/blazeface.js" "path": "src/facemesh/blazeface.js"
@ -34,7 +34,7 @@
"path": "src/facemesh/keypoints.js" "path": "src/facemesh/keypoints.js"
}, },
{ {
"path": "src/facemesh/pipeline.js" "path": "src/facemesh/facepipeline.js"
}, },
{ {
"path": "src/facemesh/uvcoords.js" "path": "src/facemesh/uvcoords.js"
@ -44,12 +44,8 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": { "src/facemesh/facepipeline.js": {
"bytes": 2507, "bytes": 14296,
"imports": []
},
"src/facemesh/pipeline.js": {
"bytes": 14262,
"imports": [ "imports": [
{ {
"path": "src/facemesh/box.js" "path": "src/facemesh/box.js"
@ -62,6 +58,10 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": {
"bytes": 2507,
"imports": []
},
"src/facemesh/triangulation.js": { "src/facemesh/triangulation.js": {
"bytes": 12940, "bytes": 12940,
"imports": [] "imports": []
@ -79,40 +79,19 @@
"imports": [] "imports": []
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytes": 2572, "bytes": 3192,
"imports": [] "imports": []
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytes": 4077, "bytes": 4484,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
} }
] ]
}, },
"src/handpose/handpose.js": { "src/handpose/handpipeline.js": {
"bytes": 1849, "bytes": 8058,
"imports": [
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/keypoints.js"
},
{
"path": "src/handpose/pipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/keypoints.js": {
"bytes": 193,
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8216,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
@ -122,12 +101,26 @@
} }
] ]
}, },
"src/handpose/handpose.js": {
"bytes": 2772,
"imports": [
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/handpipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/util.js": { "src/handpose/util.js": {
"bytes": 2488, "bytes": 3030,
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15164, "bytes": 15209,
"imports": [ "imports": [
{ {
"path": "src/facemesh/facemesh.js" "path": "src/facemesh/facemesh.js"
@ -282,7 +275,7 @@
"dist/human.esm-nobundle.js.map": { "dist/human.esm-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 605307 "bytes": 607262
}, },
"dist/human.esm-nobundle.js": { "dist/human.esm-nobundle.js": {
"imports": [], "imports": [],
@ -299,8 +292,8 @@
"src/facemesh/util.js": { "src/facemesh/util.js": {
"bytesInOutput": 1171 "bytesInOutput": 1171
}, },
"src/facemesh/pipeline.js": { "src/facemesh/facepipeline.js": {
"bytesInOutput": 5571 "bytesInOutput": 5585
}, },
"src/facemesh/uvcoords.js": { "src/facemesh/uvcoords.js": {
"bytesInOutput": 16785 "bytesInOutput": 16785
@ -318,7 +311,7 @@
"bytesInOutput": 1236 "bytesInOutput": 1236
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1104 "bytesInOutput": 1098
}, },
"src/posenet/modelBase.js": { "src/posenet/modelBase.js": {
"bytesInOutput": 455 "bytesInOutput": 455
@ -333,46 +326,43 @@
"bytesInOutput": 546 "bytesInOutput": 546
}, },
"src/posenet/keypoints.js": { "src/posenet/keypoints.js": {
"bytesInOutput": 1621 "bytesInOutput": 1620
}, },
"src/posenet/vectors.js": { "src/posenet/vectors.js": {
"bytesInOutput": 607 "bytesInOutput": 606
}, },
"src/posenet/decodePose.js": { "src/posenet/decodePose.js": {
"bytesInOutput": 1016 "bytesInOutput": 1015
}, },
"src/posenet/decodeMultiple.js": { "src/posenet/decodeMultiple.js": {
"bytesInOutput": 603 "bytesInOutput": 603
}, },
"src/posenet/util.js": { "src/posenet/util.js": {
"bytesInOutput": 1053 "bytesInOutput": 1052
}, },
"src/posenet/modelPoseNet.js": { "src/posenet/modelPoseNet.js": {
"bytesInOutput": 841 "bytesInOutput": 841
}, },
"src/posenet/posenet.js": { "src/posenet/posenet.js": {
"bytesInOutput": 459 "bytesInOutput": 458
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytesInOutput": 1376 "bytesInOutput": 1420
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytesInOutput": 2004 "bytesInOutput": 1891
},
"src/handpose/keypoints.js": {
"bytesInOutput": 155
}, },
"src/handpose/util.js": { "src/handpose/util.js": {
"bytesInOutput": 972 "bytesInOutput": 997
}, },
"src/handpose/pipeline.js": { "src/handpose/handpipeline.js": {
"bytesInOutput": 3223 "bytesInOutput": 3233
}, },
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytesInOutput": 127000 "bytesInOutput": 127000
}, },
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1014 "bytesInOutput": 1112
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11013 "bytesInOutput": 11013
@ -384,13 +374,13 @@
"bytesInOutput": 2567 "bytesInOutput": 2567
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8439 "bytesInOutput": 8467
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 211614 "bytes": 212257
} }
} }
} }

508
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

94
dist/human.esm.json vendored
View File

@ -180,7 +180,7 @@
] ]
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytes": 2568, "bytes": 2572,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -192,7 +192,7 @@
"path": "src/facemesh/keypoints.js" "path": "src/facemesh/keypoints.js"
}, },
{ {
"path": "src/facemesh/pipeline.js" "path": "src/facemesh/facepipeline.js"
}, },
{ {
"path": "src/facemesh/uvcoords.js" "path": "src/facemesh/uvcoords.js"
@ -202,12 +202,8 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": { "src/facemesh/facepipeline.js": {
"bytes": 2507, "bytes": 14296,
"imports": []
},
"src/facemesh/pipeline.js": {
"bytes": 14262,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -223,6 +219,10 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": {
"bytes": 2507,
"imports": []
},
"src/facemesh/triangulation.js": { "src/facemesh/triangulation.js": {
"bytes": 12940, "bytes": 12940,
"imports": [] "imports": []
@ -240,7 +240,7 @@
"imports": [] "imports": []
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytes": 2572, "bytes": 3192,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -248,7 +248,7 @@
] ]
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytes": 4077, "bytes": 4484,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -258,32 +258,8 @@
} }
] ]
}, },
"src/handpose/handpose.js": { "src/handpose/handpipeline.js": {
"bytes": 1849, "bytes": 8058,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/keypoints.js"
},
{
"path": "src/handpose/pipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/keypoints.js": {
"bytes": 193,
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8216,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -296,12 +272,29 @@
} }
] ]
}, },
"src/handpose/handpose.js": {
"bytes": 2772,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/handpipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/util.js": { "src/handpose/util.js": {
"bytes": 2488, "bytes": 3030,
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15164, "bytes": 15209,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -488,7 +481,7 @@
"dist/human.esm.js.map": { "dist/human.esm.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5401089 "bytes": 5403044
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -562,8 +555,8 @@
"src/facemesh/util.js": { "src/facemesh/util.js": {
"bytesInOutput": 1190 "bytesInOutput": 1190
}, },
"src/facemesh/pipeline.js": { "src/facemesh/facepipeline.js": {
"bytesInOutput": 5563 "bytesInOutput": 5577
}, },
"src/facemesh/uvcoords.js": { "src/facemesh/uvcoords.js": {
"bytesInOutput": 16786 "bytesInOutput": 16786
@ -617,25 +610,22 @@
"bytesInOutput": 474 "bytesInOutput": 474
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytesInOutput": 1362 "bytesInOutput": 1398
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytesInOutput": 2010 "bytesInOutput": 1900
},
"src/handpose/keypoints.js": {
"bytesInOutput": 156
}, },
"src/handpose/util.js": { "src/handpose/util.js": {
"bytesInOutput": 988 "bytesInOutput": 1005
}, },
"src/handpose/pipeline.js": { "src/handpose/handpipeline.js": {
"bytesInOutput": 3221 "bytesInOutput": 3231
}, },
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytesInOutput": 127001 "bytesInOutput": 127001
}, },
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 992 "bytesInOutput": 1090
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11014 "bytesInOutput": 11014
@ -647,13 +637,13 @@
"bytesInOutput": 2568 "bytesInOutput": 2568
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8455 "bytesInOutput": 8484
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 0 "bytesInOutput": 0
} }
}, },
"bytes": 1273160 "bytes": 1273801
} }
} }
} }

508
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

94
dist/human.json vendored
View File

@ -180,7 +180,7 @@
] ]
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytes": 2568, "bytes": 2572,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -192,7 +192,7 @@
"path": "src/facemesh/keypoints.js" "path": "src/facemesh/keypoints.js"
}, },
{ {
"path": "src/facemesh/pipeline.js" "path": "src/facemesh/facepipeline.js"
}, },
{ {
"path": "src/facemesh/uvcoords.js" "path": "src/facemesh/uvcoords.js"
@ -202,12 +202,8 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": { "src/facemesh/facepipeline.js": {
"bytes": 2507, "bytes": 14296,
"imports": []
},
"src/facemesh/pipeline.js": {
"bytes": 14262,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -223,6 +219,10 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": {
"bytes": 2507,
"imports": []
},
"src/facemesh/triangulation.js": { "src/facemesh/triangulation.js": {
"bytes": 12940, "bytes": 12940,
"imports": [] "imports": []
@ -240,7 +240,7 @@
"imports": [] "imports": []
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytes": 2572, "bytes": 3192,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -248,7 +248,7 @@
] ]
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytes": 4077, "bytes": 4484,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -258,32 +258,8 @@
} }
] ]
}, },
"src/handpose/handpose.js": { "src/handpose/handpipeline.js": {
"bytes": 1849, "bytes": 8058,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/keypoints.js"
},
{
"path": "src/handpose/pipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/keypoints.js": {
"bytes": 193,
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8216,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -296,12 +272,29 @@
} }
] ]
}, },
"src/handpose/handpose.js": {
"bytes": 2772,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
},
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/handpipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/util.js": { "src/handpose/util.js": {
"bytes": 2488, "bytes": 3030,
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15164, "bytes": 15209,
"imports": [ "imports": [
{ {
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js" "path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -488,7 +481,7 @@
"dist/human.js.map": { "dist/human.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 5401085 "bytes": 5403040
}, },
"dist/human.js": { "dist/human.js": {
"imports": [], "imports": [],
@ -562,8 +555,8 @@
"src/facemesh/util.js": { "src/facemesh/util.js": {
"bytesInOutput": 1190 "bytesInOutput": 1190
}, },
"src/facemesh/pipeline.js": { "src/facemesh/facepipeline.js": {
"bytesInOutput": 5563 "bytesInOutput": 5577
}, },
"src/facemesh/uvcoords.js": { "src/facemesh/uvcoords.js": {
"bytesInOutput": 16786 "bytesInOutput": 16786
@ -617,25 +610,22 @@
"bytesInOutput": 474 "bytesInOutput": 474
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytesInOutput": 1362 "bytesInOutput": 1398
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytesInOutput": 2010 "bytesInOutput": 1900
},
"src/handpose/keypoints.js": {
"bytesInOutput": 156
}, },
"src/handpose/util.js": { "src/handpose/util.js": {
"bytesInOutput": 988 "bytesInOutput": 1005
}, },
"src/handpose/pipeline.js": { "src/handpose/handpipeline.js": {
"bytesInOutput": 3221 "bytesInOutput": 3231
}, },
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytesInOutput": 127001 "bytesInOutput": 127001
}, },
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 992 "bytesInOutput": 1090
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11014 "bytesInOutput": 11014
@ -647,10 +637,10 @@
"bytesInOutput": 2567 "bytesInOutput": 2567
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8493 "bytesInOutput": 8522
} }
}, },
"bytes": 1273205 "bytes": 1273846
} }
} }
} }

32
dist/human.node-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

508
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

96
dist/human.node.json vendored
View File

@ -25,7 +25,7 @@
"imports": [] "imports": []
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytes": 2568, "bytes": 2572,
"imports": [ "imports": [
{ {
"path": "src/facemesh/blazeface.js" "path": "src/facemesh/blazeface.js"
@ -34,7 +34,7 @@
"path": "src/facemesh/keypoints.js" "path": "src/facemesh/keypoints.js"
}, },
{ {
"path": "src/facemesh/pipeline.js" "path": "src/facemesh/facepipeline.js"
}, },
{ {
"path": "src/facemesh/uvcoords.js" "path": "src/facemesh/uvcoords.js"
@ -44,12 +44,8 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": { "src/facemesh/facepipeline.js": {
"bytes": 2507, "bytes": 14296,
"imports": []
},
"src/facemesh/pipeline.js": {
"bytes": 14262,
"imports": [ "imports": [
{ {
"path": "src/facemesh/box.js" "path": "src/facemesh/box.js"
@ -62,6 +58,10 @@
} }
] ]
}, },
"src/facemesh/keypoints.js": {
"bytes": 2507,
"imports": []
},
"src/facemesh/triangulation.js": { "src/facemesh/triangulation.js": {
"bytes": 12940, "bytes": 12940,
"imports": [] "imports": []
@ -79,40 +79,19 @@
"imports": [] "imports": []
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytes": 2572, "bytes": 3192,
"imports": [] "imports": []
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytes": 4077, "bytes": 4484,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
} }
] ]
}, },
"src/handpose/handpose.js": { "src/handpose/handpipeline.js": {
"bytes": 1849, "bytes": 8058,
"imports": [
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/keypoints.js"
},
{
"path": "src/handpose/pipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/keypoints.js": {
"bytes": 193,
"imports": []
},
"src/handpose/pipeline.js": {
"bytes": 8216,
"imports": [ "imports": [
{ {
"path": "src/handpose/box.js" "path": "src/handpose/box.js"
@ -122,12 +101,26 @@
} }
] ]
}, },
"src/handpose/handpose.js": {
"bytes": 2772,
"imports": [
{
"path": "src/handpose/handdetector.js"
},
{
"path": "src/handpose/handpipeline.js"
},
{
"path": "src/handpose/anchors.js"
}
]
},
"src/handpose/util.js": { "src/handpose/util.js": {
"bytes": 2488, "bytes": 3030,
"imports": [] "imports": []
}, },
"src/human.js": { "src/human.js": {
"bytes": 15164, "bytes": 15209,
"imports": [ "imports": [
{ {
"path": "src/facemesh/facemesh.js" "path": "src/facemesh/facemesh.js"
@ -282,7 +275,7 @@
"dist/human.node-nobundle.js.map": { "dist/human.node-nobundle.js.map": {
"imports": [], "imports": [],
"inputs": {}, "inputs": {},
"bytes": 620872 "bytes": 622873
}, },
"dist/human.node-nobundle.js": { "dist/human.node-nobundle.js": {
"imports": [], "imports": [],
@ -299,14 +292,14 @@
"src/facemesh/util.js": { "src/facemesh/util.js": {
"bytesInOutput": 1171 "bytesInOutput": 1171
}, },
"src/facemesh/pipeline.js": { "src/facemesh/facepipeline.js": {
"bytesInOutput": 5571 "bytesInOutput": 5585
}, },
"src/facemesh/uvcoords.js": { "src/facemesh/uvcoords.js": {
"bytesInOutput": 16785 "bytesInOutput": 16785
}, },
"src/facemesh/triangulation.js": { "src/facemesh/triangulation.js": {
"bytesInOutput": 9990 "bytesInOutput": 9989
}, },
"src/facemesh/facemesh.js": { "src/facemesh/facemesh.js": {
"bytesInOutput": 1254 "bytesInOutput": 1254
@ -318,7 +311,7 @@
"bytesInOutput": 1236 "bytesInOutput": 1236
}, },
"src/emotion/emotion.js": { "src/emotion/emotion.js": {
"bytesInOutput": 1104 "bytesInOutput": 1098
}, },
"src/posenet/modelBase.js": { "src/posenet/modelBase.js": {
"bytesInOutput": 455 "bytesInOutput": 455
@ -354,43 +347,40 @@
"bytesInOutput": 459 "bytesInOutput": 459
}, },
"src/handpose/box.js": { "src/handpose/box.js": {
"bytesInOutput": 1376 "bytesInOutput": 1419
}, },
"src/handpose/handdetector.js": { "src/handpose/handdetector.js": {
"bytesInOutput": 2004 "bytesInOutput": 1891
},
"src/handpose/keypoints.js": {
"bytesInOutput": 155
}, },
"src/handpose/util.js": { "src/handpose/util.js": {
"bytesInOutput": 972 "bytesInOutput": 996
}, },
"src/handpose/pipeline.js": { "src/handpose/handpipeline.js": {
"bytesInOutput": 3223 "bytesInOutput": 3233
}, },
"src/handpose/anchors.js": { "src/handpose/anchors.js": {
"bytesInOutput": 127000 "bytesInOutput": 127000
}, },
"src/handpose/handpose.js": { "src/handpose/handpose.js": {
"bytesInOutput": 1014 "bytesInOutput": 1112
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytesInOutput": 11013 "bytesInOutput": 11013
}, },
"config.js": { "config.js": {
"bytesInOutput": 1272 "bytesInOutput": 1271
}, },
"package.json": { "package.json": {
"bytesInOutput": 2567 "bytesInOutput": 2567
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 29 "bytesInOutput": 28
}, },
"src/human.js": { "src/human.js": {
"bytesInOutput": 8439 "bytesInOutput": 8467
} }
}, },
"bytes": 211622 "bytes": 212264
} }
} }
} }

30
package-lock.json generated
View File

@ -524,7 +524,8 @@
"callsites": { "callsites": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
"dev": true
}, },
"chalk": { "chalk": {
"version": "4.1.0", "version": "4.1.0",
@ -1407,6 +1408,7 @@
"version": "3.2.2", "version": "3.2.2",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz",
"integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==",
"dev": true,
"requires": { "requires": {
"parent-module": "^1.0.0", "parent-module": "^1.0.0",
"resolve-from": "^4.0.0" "resolve-from": "^4.0.0"
@ -1570,8 +1572,7 @@
"jsonc-parser": { "jsonc-parser": {
"version": "2.3.1", "version": "2.3.1",
"resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-2.3.1.tgz", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-2.3.1.tgz",
"integrity": "sha512-H8jvkz1O50L3dMZCsLqiuB2tA7muqbSg1AtGEkN0leAqGjsUzDJir3Zwr02BhqdcITPg3ei3mZ+HjMocAknhhg==", "integrity": "sha512-H8jvkz1O50L3dMZCsLqiuB2tA7muqbSg1AtGEkN0leAqGjsUzDJir3Zwr02BhqdcITPg3ei3mZ+HjMocAknhhg=="
"dev": true
}, },
"levn": { "levn": {
"version": "0.4.1", "version": "0.4.1",
@ -1961,6 +1962,7 @@
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
"dev": true,
"requires": { "requires": {
"callsites": "^3.0.0" "callsites": "^3.0.0"
} }
@ -2125,7 +2127,8 @@
"resolve-from": { "resolve-from": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==" "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true
}, },
"rimraf": { "rimraf": {
"version": "3.0.2", "version": "3.0.2",
@ -2498,10 +2501,9 @@
} }
}, },
"vscode-json-languageservice": { "vscode-json-languageservice": {
"version": "3.9.1", "version": "3.10.0",
"resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-3.9.1.tgz", "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-3.10.0.tgz",
"integrity": "sha512-oJkknkdCVitQ5XPSRa0weHjUxt8eSCptaL+MBQQlRsa6Nb8XnEY0S5wYnLUFHzEvKzwt01/LKk8LdOixWEXkNA==", "integrity": "sha512-8IvuRSQnjznu+obqy6Dy4S4H68Ke7a3Kb+A0FcdctyAMAWEnrORpCpMOMqEYiPLm/OTYLVWJ7ql3qToDTozu4w==",
"dev": true,
"requires": { "requires": {
"jsonc-parser": "^2.3.1", "jsonc-parser": "^2.3.1",
"vscode-languageserver-textdocument": "^1.0.1", "vscode-languageserver-textdocument": "^1.0.1",
@ -2513,26 +2515,22 @@
"vscode-languageserver-textdocument": { "vscode-languageserver-textdocument": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.1.tgz", "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.1.tgz",
"integrity": "sha512-UIcJDjX7IFkck7cSkNNyzIz5FyvpQfY7sdzVy+wkKN/BLaD4DQ0ppXQrKePomCxTS7RrolK1I0pey0bG9eh8dA==", "integrity": "sha512-UIcJDjX7IFkck7cSkNNyzIz5FyvpQfY7sdzVy+wkKN/BLaD4DQ0ppXQrKePomCxTS7RrolK1I0pey0bG9eh8dA=="
"dev": true
}, },
"vscode-languageserver-types": { "vscode-languageserver-types": {
"version": "3.16.0-next.2", "version": "3.16.0-next.2",
"resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0-next.2.tgz", "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0-next.2.tgz",
"integrity": "sha512-QjXB7CKIfFzKbiCJC4OWC8xUncLsxo19FzGVp/ADFvvi87PlmBSCAtZI5xwGjF5qE0xkLf0jjKUn3DzmpDP52Q==", "integrity": "sha512-QjXB7CKIfFzKbiCJC4OWC8xUncLsxo19FzGVp/ADFvvi87PlmBSCAtZI5xwGjF5qE0xkLf0jjKUn3DzmpDP52Q=="
"dev": true
}, },
"vscode-nls": { "vscode-nls": {
"version": "5.0.0", "version": "5.0.0",
"resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.0.0.tgz", "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.0.0.tgz",
"integrity": "sha512-u0Lw+IYlgbEJFF6/qAqG2d1jQmJl0eyAGJHoAJqr2HT4M2BNuQYSEiSE75f52pXHSJm8AlTjnLLbBFPrdz2hpA==", "integrity": "sha512-u0Lw+IYlgbEJFF6/qAqG2d1jQmJl0eyAGJHoAJqr2HT4M2BNuQYSEiSE75f52pXHSJm8AlTjnLLbBFPrdz2hpA=="
"dev": true
}, },
"vscode-uri": { "vscode-uri": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz",
"integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==", "integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A=="
"dev": true
}, },
"which": { "which": {
"version": "2.0.2", "version": "2.0.2",

2
src/facemesh/facemesh.js
View File

@ -1,7 +1,7 @@
const tf = require('@tensorflow/tfjs'); const tf = require('@tensorflow/tfjs');
const blazeface = require('./blazeface'); const blazeface = require('./blazeface');
const keypoints = require('./keypoints'); const keypoints = require('./keypoints');
const pipe = require('./pipeline'); const pipe = require('./facepipeline');
const uv_coords = require('./uvcoords'); const uv_coords = require('./uvcoords');
const triangulation = require('./triangulation').default; const triangulation = require('./triangulation').default;

2
src/facemesh/pipeline.js → src/facemesh/facepipeline.js
View File

@ -134,7 +134,7 @@ class Pipeline {
this.runsWithoutFaceDetector++; this.runsWithoutFaceDetector++;
if (this.shouldUpdateRegionsOfInterest()) { if (this.shouldUpdateRegionsOfInterest()) {
const detector = await this.boundingBoxDetector.getBoundingBoxes(input); const detector = await this.boundingBoxDetector.getBoundingBoxes(input);
if (detector.boxes.length === 0) { if (!detector || !detector.boxes || (detector.boxes.length === 0)) {
this.regionsOfInterest = []; this.regionsOfInterest = [];
return null; return null;
} }

50
src/handpose/box.js
View File

@ -1,3 +1,19 @@
/**
* @license
* Copyright 2020 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================================
*/
const tf = require('@tensorflow/tfjs'); const tf = require('@tensorflow/tfjs');
function getBoxSize(box) { function getBoxSize(box) {
@ -6,35 +22,32 @@ function getBoxSize(box) {
Math.abs(box.endPoint[1] - box.startPoint[1]), Math.abs(box.endPoint[1] - box.startPoint[1]),
]; ];
} }
exports.getBoxSize = getBoxSize;
function getBoxCenter(box) { function getBoxCenter(box) {
return [ return [
box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2, box.startPoint[0] + (box.endPoint[0] - box.startPoint[0]) / 2,
box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2, box.startPoint[1] + (box.endPoint[1] - box.startPoint[1]) / 2,
]; ];
} }
exports.getBoxCenter = getBoxCenter;
function cutBoxFromImageAndResize(box, image, cropSize) { function cutBoxFromImageAndResize(box, image, cropSize) {
const h = image.shape[1]; const h = image.shape[1];
const w = image.shape[2]; const w = image.shape[2];
const boxes = [[ const boxes = [[
box.startPoint[1] / h, box.startPoint[0] / w, box.endPoint[1] / h, box.startPoint[1] / h,
box.startPoint[0] / w,
box.endPoint[1] / h,
box.endPoint[0] / w, box.endPoint[0] / w,
]]; ]];
return tf.image.cropAndResize(image, boxes, [0], cropSize); return tf.image.cropAndResize(image, boxes, [0], cropSize);
} }
exports.cutBoxFromImageAndResize = cutBoxFromImageAndResize;
function scaleBoxCoordinates(box, factor) { function scaleBoxCoordinates(box, factor) {
const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]]; const startPoint = [box.startPoint[0] * factor[0], box.startPoint[1] * factor[1]];
const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]]; const endPoint = [box.endPoint[0] * factor[0], box.endPoint[1] * factor[1]];
const palmLandmarks = box.palmLandmarks.map((coord) => [coord[0] * factor[0], coord[1] * factor[1]]); const palmLandmarks = box.palmLandmarks.map((coord) => {
const scaledCoord = [coord[0] * factor[0], coord[1] * factor[1]];
return scaledCoord;
});
return { startPoint, endPoint, palmLandmarks }; return { startPoint, endPoint, palmLandmarks };
} }
exports.scaleBoxCoordinates = scaleBoxCoordinates;
function enlargeBox(box, factor = 1.5) { function enlargeBox(box, factor = 1.5) {
const center = getBoxCenter(box); const center = getBoxCenter(box);
const size = getBoxSize(box); const size = getBoxSize(box);
@ -43,8 +56,6 @@ function enlargeBox(box, factor = 1.5) {
const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]]; const endPoint = [center[0] + newHalfSize[0], center[1] + newHalfSize[1]];
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks }; return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
} }
exports.enlargeBox = enlargeBox;
function squarifyBox(box) { function squarifyBox(box) {
const centers = getBoxCenter(box); const centers = getBoxCenter(box);
const size = getBoxSize(box); const size = getBoxSize(box);
@ -54,15 +65,22 @@ function squarifyBox(box) {
const endPoint = [centers[0] + halfSize, centers[1] + halfSize]; const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks }; return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
} }
exports.squarifyBox = squarifyBox;
function shiftBox(box, shiftFactor) { function shiftBox(box, shiftFactor) {
const boxSize = [ const boxSize = [
box.endPoint[0] - box.startPoint[0], box.endPoint[1] - box.startPoint[1], box.endPoint[0] - box.startPoint[0],
box.endPoint[1] - box.startPoint[1],
]; ];
const shiftVector = [boxSize[0] * shiftFactor[0], boxSize[1] * shiftFactor[1]]; const shiftVector = [boxSize[0] * shiftFactor[0], boxSize[1] * shiftFactor[1]];
const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]]; const startPoint = [box.startPoint[0] + shiftVector[0], box.startPoint[1] + shiftVector[1]];
const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]]; const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]];
return { startPoint, endPoint, palmLandmarks: box.palmLandmarks }; return { startPoint, endPoint, palmLandmarks: box.palmLandmarks };
} }
exports.shiftBox = shiftBox; export {
cutBoxFromImageAndResize,
enlargeBox,
getBoxCenter,
getBoxSize,
scaleBoxCoordinates,
shiftBox,
squarifyBox,
};

113
src/handpose/handdetector.js
View File

@ -1,15 +1,32 @@
/**
* @license
* Copyright 2020 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================================
*/
const tf = require('@tensorflow/tfjs'); const tf = require('@tensorflow/tfjs');
const bounding = require('./box'); const box = require('./box');
class HandDetector { class HandDetector {
constructor(model, anchors, config) { constructor(model, inputSize, anchorsAnnotated) {
this.model = model; this.model = model;
this.width = config.inputSize; this.width = inputSize;
this.height = config.inputSize; this.height = inputSize;
this.anchors = anchors.map((anchor) => [anchor.x_center, anchor.y_center]); this.anchors = anchorsAnnotated.map((anchor) => [anchor.x_center, anchor.y_center]);
this.anchorsTensor = tf.tensor2d(this.anchors); this.anchorsTensor = tf.tensor2d(this.anchors);
this.inputSizeTensor = tf.tensor1d([config.inputSize, config.inputSize]); this.inputSizeTensor = tf.tensor1d([inputSize, inputSize]);
this.doubleInputSizeTensor = tf.tensor1d([config.inputSize * 2, config.inputSize * 2]); this.doubleInputSizeTensor = tf.tensor1d([inputSize * 2, inputSize * 2]);
} }
normalizeBoxes(boxes) { normalizeBoxes(boxes) {
@ -31,59 +48,59 @@ class HandDetector {
}); });
} }
async getBoundingBoxes(input) { async getBoundingBoxes(input, config) {
const batchedPrediction = this.model.predict(input); const normalizedInput = tf.tidy(() => tf.mul(tf.sub(input, 0.5), 2));
const batchedPrediction = this.model.predict(normalizedInput);
const prediction = batchedPrediction.squeeze(); const prediction = batchedPrediction.squeeze();
// Regression score for each anchor point.
const scores = tf.tidy(() => tf.sigmoid(tf.slice(prediction, [0, 0], [-1, 1])).squeeze()); const scores = tf.tidy(() => tf.sigmoid(tf.slice(prediction, [0, 0], [-1, 1])).squeeze());
// Bounding box for each anchor point.
const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]); const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes); const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.maxHands, this.iouThreshold, this.scoreThreshold); const boxesWithHandsTensor = tf.image.nonMaxSuppression(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
const boxesWithHands = boxesWithHandsTensor.arraySync(); const boxesWithHands = boxesWithHandsTensor.arraySync();
const detectedHands = tf.tidy(() => { const toDispose = [
const detectedBoxes = []; normalizedInput,
for (const i in boxesWithHands) { batchedPrediction,
const boxIndex = boxesWithHands[i]; boxesWithHandsTensor,
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]); prediction,
const rawPalmLandmarks = tf.slice(prediction, [boxIndex, 5], [1, 14]); boxes,
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2])); rawBoxes,
detectedBoxes.push({ boxes: matchingBox, palmLandmarks }); scores,
} ];
return detectedBoxes; if (boxesWithHands.length === 0) {
}); toDispose.forEach((tensor) => tensor.dispose());
[batchedPrediction, boxesWithHandsTensor, prediction, boxes, rawBoxes, scores].forEach((tensor) => tensor.dispose()); return null;
return detectedHands; }
const hands = [];
for (const boxIndex of boxesWithHands) {
const matchingBox = tf.slice(boxes, [boxIndex, 0], [1, -1]);
const rawPalmLandmarks = tf.slice(prediction, [boxIndex, 5], [1, 14]);
const palmLandmarks = tf.tidy(() => this.normalizeLandmarks(rawPalmLandmarks, boxIndex).reshape([-1, 2]));
rawPalmLandmarks.dispose();
hands.push({ boxes: matchingBox, palmLandmarks });
}
toDispose.forEach((tensor) => tensor.dispose());
return hands;
} }
/**
* Returns a Box identifying the bounding box of a hand within the image.
* Returns null if there is no hand in the image.
*
* @param input The image to classify.
*/
async estimateHandBounds(input, config) { async estimateHandBounds(input, config) {
this.iouThreshold = config.iouThreshold; const inputHeight = input.shape[1];
this.scoreThreshold = config.scoreThreshold; const inputWidth = input.shape[2];
this.maxHands = config.maxHands; const image = tf.tidy(() => input.resizeBilinear([this.width, this.height]).div(255));
const resized = input.resizeBilinear([this.width, this.height]); const predictions = await this.getBoundingBoxes(image, config);
const divided = resized.mul([1 / 127.5]); if (!predictions || predictions.length === 0) {
const image = divided.sub(0.5); image.dispose();
resized.dispose(); return null;
divided.dispose(); }
const predictions = await this.getBoundingBoxes(image);
image.dispose();
if (!predictions || (predictions.length === 0)) return null;
const hands = []; const hands = [];
for (const i in predictions) { for (const prediction of predictions) {
const prediction = predictions[i]; const boundingBoxes = prediction.boxes.arraySync();
const boundingBoxes = prediction.boxes.dataSync(); const startPoint = boundingBoxes[0].slice(0, 2);
const startPoint = [boundingBoxes[0], boundingBoxes[1]]; const endPoint = boundingBoxes[0].slice(2, 4);
const endPoint = [boundingBoxes[2], boundingBoxes[3]];
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = prediction.palmLandmarks.arraySync();
image.dispose();
prediction.boxes.dispose(); prediction.boxes.dispose();
prediction.palmLandmarks.dispose(); prediction.palmLandmarks.dispose();
hands.push(bounding.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [input.shape[2] / this.width, input.shape[1] / this.height])); hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / this.width, inputHeight / this.height]));
} }
return hands; return hands;
} }

184
src/handpose/handpipeline.js Normal file
View File

@ -0,0 +1,184 @@
/**
* @license
* Copyright 2020 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================================
*/
const tf = require('@tensorflow/tfjs');
const box = require('./box');
const util = require('./util');
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8;
const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
const PALM_BOX_ENLARGE_FACTOR = 3;
const HAND_BOX_SHIFT_VECTOR = [0, -0.1];
const HAND_BOX_ENLARGE_FACTOR = 1.65;
const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2];
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
class HandPipeline {
constructor(boundingBoxDetector, meshDetector, inputSize) {
this.boundingBoxDetector = boundingBoxDetector;
this.meshDetector = meshDetector;
this.inputSize = inputSize;
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.maxHandsNumber = 1;
this.skipFrames = 0;
}
getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
const rotatedPalmLandmarks = palmLandmarks.map((coord) => {
const homogeneousCoordinate = [...coord, 1];
return util.rotatePoint(homogeneousCoordinate, rotationMatrix);
});
const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks);
return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR);
}
getBoxForHandLandmarks(landmarks) {
const boundingBox = this.calculateLandmarksBoundingBox(landmarks);
const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
const palmLandmarks = [];
for (let i = 0; i < PALM_LANDMARK_IDS.length; i++) {
palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
}
boxAroundHand.palmLandmarks = palmLandmarks;
return boxAroundHand;
}
transformRawCoords(rawCoords, box2, angle, rotationMatrix) {
const boxSize = box.getBoxSize(box2);
const scaleFactor = [boxSize[0] / this.inputSize, boxSize[1] / this.inputSize];
const coordsScaled = rawCoords.map((coord) => [
scaleFactor[0] * (coord[0] - this.inputSize / 2),
scaleFactor[1] * (coord[1] - this.inputSize / 2),
coord[2],
]);
const coordsRotationMatrix = util.buildRotationMatrix(angle, [0, 0]);
const coordsRotated = coordsScaled.map((coord) => {
const rotated = util.rotatePoint(coord, coordsRotationMatrix);
return [...rotated, coord[2]];
});
const inverseRotationMatrix = util.invertTransformMatrix(rotationMatrix);
const boxCenter = [...box.getBoxCenter(box2), 1];
const originalBoxCenter = [
util.dot(boxCenter, inverseRotationMatrix[0]),
util.dot(boxCenter, inverseRotationMatrix[1]),
];
return coordsRotated.map((coord) => [
coord[0] + originalBoxCenter[0],
coord[1] + originalBoxCenter[1],
coord[2],
]);
}
async estimateHands(image, config) {
this.skipFrames = config.skipFrames;
const useFreshBox = this.shouldUpdateRegionsOfInterest();
if (useFreshBox) {
const boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
this.regionsOfInterest = [];
if (!boundingBoxPredictions || boundingBoxPredictions.length === 0) {
image.dispose();
return null;
}
for (const boundingBoxPrediction of boundingBoxPredictions) {
this.regionsOfInterest.push(boundingBoxPrediction);
}
this.runsWithoutHandDetector = 0;
} else {
this.runsWithoutHandDetector++;
}
const hands = [];
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i];
if (!currentBox) continue;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
const palmCenter = box.getBoxCenter(currentBox);
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.meshDetector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const confidenceValue = confidence.dataSync()[0];
confidence.dispose();
if (confidenceValue < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = null;
return null;
}
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateRegionsOfInterest(nextBoundingBox, i);
const result = {
landmarks: coords,
handInViewConfidence: confidenceValue,
boundingBox: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint,
},
};
hands.push(result);
}
return hands;
}
// eslint-disable-next-line class-methods-use-this
calculateLandmarksBoundingBox(landmarks) {
const xs = landmarks.map((d) => d[0]);
const ys = landmarks.map((d) => d[1]);
const startPoint = [Math.min(...xs), Math.min(...ys)];
const endPoint = [Math.max(...xs), Math.max(...ys)];
return { startPoint, endPoint };
}
updateRegionsOfInterest(newBox, i) {
const previousBox = this.regionsOfInterest[i];
let iou = 0;
if (previousBox != null && previousBox.startPoint != null) {
const [boxStartX, boxStartY] = newBox.startPoint;
const [boxEndX, boxEndY] = newBox.endPoint;
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;
const [previousBoxEndX, previousBoxEndY] = previousBox.endPoint;
const xStartMax = Math.max(boxStartX, previousBoxStartX);
const yStartMax = Math.max(boxStartY, previousBoxStartY);
const xEndMin = Math.min(boxEndX, previousBoxEndX);
const yEndMin = Math.min(boxEndY, previousBoxEndY);
const intersection = (xEndMin - xStartMax) * (yEndMin - yStartMax);
const boxArea = (boxEndX - boxStartX) * (boxEndY - boxStartY);
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
iou = intersection / (boxArea + previousBoxArea - intersection);
}
this.regionsOfInterest[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox;
}
shouldUpdateRegionsOfInterest() {
console.log(this.regionsOfInterest.length, this.runsWithoutHandDetector, !this.regionsOfInterest || (this.regionsOfInterest.length === 0) || (this.runsWithoutHandDetector >= this.skipFrames));
return !this.regionsOfInterest || (this.regionsOfInterest.length === 0) || (this.runsWithoutHandDetector >= this.skipFrames);
}
}
exports.HandPipeline = HandPipeline;

61
src/handpose/handpose.js
View File

@ -1,30 +1,54 @@
/**
* @license
* Copyright 2020 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================================
*/
const tf = require('@tensorflow/tfjs'); const tf = require('@tensorflow/tfjs');
const hand = require('./handdetector'); const handdetector = require('./handdetector');
const keypoints = require('./keypoints'); const pipeline = require('./handpipeline');
const pipe = require('./pipeline'); const anchors = require('./anchors');
const anchors = require('./anchors.js');
const MESH_ANNOTATIONS = {
thumb: [1, 2, 3, 4],
indexFinger: [5, 6, 7, 8],
middleFinger: [9, 10, 11, 12],
ringFinger: [13, 14, 15, 16],
pinky: [17, 18, 19, 20],
palmBase: [0],
};
class HandPose { class HandPose {
constructor(pipeline) { constructor(pipe) {
this.pipeline = pipeline; this.pipeline = pipe;
}
static getAnnotations() {
return MESH_ANNOTATIONS;
} }
async estimateHands(input, config) { async estimateHands(input, config) {
this.skipFrames = config.skipFrames;
this.detectionConfidence = config.minConfidence;
this.maxHands = config.maxHands;
const predictions = await this.pipeline.estimateHands(input, config); const predictions = await this.pipeline.estimateHands(input, config);
if (!predictions) return [];
const hands = []; const hands = [];
if (!predictions) return hands;
for (const prediction of predictions) { for (const prediction of predictions) {
if (!prediction) return [];
const annotations = {}; const annotations = {};
for (const key of Object.keys(keypoints.MESH_ANNOTATIONS)) { for (const key of Object.keys(MESH_ANNOTATIONS)) {
annotations[key] = keypoints.MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]); annotations[key] = MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
} }
hands.push({ hands.push({
confidence: prediction.confidence || 0, confidence: prediction.handInViewConfidence,
box: prediction.box ? [prediction.box.topLeft[0], prediction.box.topLeft[1], prediction.box.bottomRight[0] - prediction.box.topLeft[0], prediction.box.bottomRight[1] - prediction.box.topLeft[1]] : 0, box: prediction.boundingBox ? [prediction.boundingBox.topLeft[0], prediction.boundingBox.topLeft[1], prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0], prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1]] : 0,
landmarks: prediction.landmarks, landmarks: prediction.landmarks,
annotations, annotations,
}); });
@ -35,13 +59,14 @@ class HandPose {
exports.HandPose = HandPose; exports.HandPose = HandPose;
async function load(config) { async function load(config) {
// maxContinuousChecks = Infinity, detectionConfidence = 0.8, iouThreshold = 0.3, scoreThreshold = 0.5
const [handDetectorModel, handPoseModel] = await Promise.all([ const [handDetectorModel, handPoseModel] = await Promise.all([
tf.loadGraphModel(config.detector.modelPath, { fromTFHub: config.detector.modelPath.includes('tfhub.dev') }), tf.loadGraphModel(config.detector.modelPath, { fromTFHub: config.detector.modelPath.includes('tfhub.dev') }),
tf.loadGraphModel(config.skeleton.modelPath, { fromTFHub: config.skeleton.modelPath.includes('tfhub.dev') }), tf.loadGraphModel(config.skeleton.modelPath, { fromTFHub: config.skeleton.modelPath.includes('tfhub.dev') }),
]); ]);
const detector = new hand.HandDetector(handDetectorModel, anchors.anchors, config); const detector = new handdetector.HandDetector(handDetectorModel, config.inputSize, anchors.anchors);
const pipeline = new pipe.HandPipeline(detector, handPoseModel, config); const pipe = new pipeline.HandPipeline(detector, handPoseModel, config.inputSize);
const handpose = new HandPose(pipeline); const handpose = new HandPose(pipe);
return handpose; return handpose;
} }
exports.load = load; exports.load = load;

8
src/handpose/keypoints.js
View File

@ -1,8 +0,0 @@
exports.MESH_ANNOTATIONS = {
thumb: [1, 2, 3, 4],
indexFinger: [5, 6, 7, 8],
middleFinger: [9, 10, 11, 12],
ringFinger: [13, 14, 15, 16],
pinky: [17, 18, 19, 20],
palmBase: [0],
};

177
src/handpose/pipeline.js
View File

@ -1,177 +0,0 @@
const tf = require('@tensorflow/tfjs');
const bounding = require('./box');
const util = require('./util');
const UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD = 0.8;
const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
const HAND_BOX_SHIFT_VECTOR = [0, -0.1];
const HAND_BOX_ENLARGE_FACTOR = 1.65;
const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2];
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
// The Pipeline coordinates between the bounding box and skeleton models.
class HandPipeline {
constructor(boundingBoxDetector, detector, config) {
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.boundingBoxDetector = boundingBoxDetector;
this.detector = detector;
this.meshWidth = config.inputSize;
this.meshHeight = config.inputSize;
this.enlargeFactor = config.enlargeFactor;
}
// Get the bounding box surrounding the hand, given palm landmarks.
getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
const rotatedPalmLandmarks = palmLandmarks.map((coord) => {
const homogeneousCoordinate = [...coord, 1];
return util.rotatePoint(homogeneousCoordinate, rotationMatrix);
});
const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks);
// boxAroundPalm only surrounds the palm - therefore we shift it
// upwards so it will capture fingers once enlarged + squarified.
return bounding.enlargeBox(bounding.squarifyBox(bounding.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), this.enlargeFactor);
}
// Get the bounding box surrounding the hand, given all hand landmarks.
getBoxForHandLandmarks(landmarks) {
// The MediaPipe hand mesh model is trained on hands with empty space
// around them, so we still need to shift / enlarge boxAroundHand even
// though it surrounds the entire hand.
const boundingBox = this.calculateLandmarksBoundingBox(landmarks);
const boxAroundHand = bounding.enlargeBox(bounding.squarifyBox(bounding.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
const palmLandmarks = [];
for (let i = 0; i < PALM_LANDMARK_IDS.length; i++) {
palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
}
boxAroundHand.palmLandmarks = palmLandmarks;
return boxAroundHand;
}
// Scale, rotate, and translate raw keypoints from the model so they map to
// the input coordinates.
transformRawCoords(rawCoords, box, angle, rotationMatrix) {
const boxSize = bounding.getBoxSize(box);
const scaleFactor = [boxSize[0] / this.meshWidth, boxSize[1] / this.meshHeight];
const coordsScaled = rawCoords.map((coord) => [
scaleFactor[0] * (coord[0] - this.meshWidth / 2),
scaleFactor[1] * (coord[1] - this.meshHeight / 2), coord[2],
]);
const coordsRotationMatrix = util.buildRotationMatrix(angle, [0, 0]);
const coordsRotated = coordsScaled.map((coord) => {
const rotated = util.rotatePoint(coord, coordsRotationMatrix);
return [...rotated, coord[2]];
});
const inverseRotationMatrix = util.invertTransformMatrix(rotationMatrix);
const boxCenter = [...bounding.getBoxCenter(box), 1];
const originalBoxCenter = [
util.dot(boxCenter, inverseRotationMatrix[0]),
util.dot(boxCenter, inverseRotationMatrix[1]),
];
return coordsRotated.map((coord) => [
coord[0] + originalBoxCenter[0], coord[1] + originalBoxCenter[1],
coord[2],
]);
}
async estimateHands(image, config) {
this.skipFrames = config.skipFrames;
this.detectionConfidence = config.minConfidence;
this.maxHands = config.maxHands;
this.runsWithoutHandDetector++;
const useFreshBox = this.shouldUpdateRegionsOfInterest();
if (useFreshBox === true) {
const boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
this.regionsOfInterest = [];
for (const i in boundingBoxPredictions) {
this.updateRegionsOfInterest(boundingBoxPredictions[i], true /* force update */, i);
}
this.runsWithoutHandDetector = 0;
}
// Rotate input so the hand is vertically oriented.
const hands = [];
if (!this.regionsOfInterest) return hands;
for (const i in this.regionsOfInterest) {
const currentBox = this.regionsOfInterest[i] ? this.regionsOfInterest[i][0] : null;
if (!currentBox) return hands;
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
const palmCenter = bounding.getBoxCenter(currentBox);
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
const box = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
const croppedInput = bounding.cutBoxFromImageAndResize(box, rotatedImage, [this.meshWidth, this.meshHeight]);
const handImage = croppedInput.div(255);
croppedInput.dispose();
rotatedImage.dispose();
const prediction = this.detector.predict(handImage);
const [confidence, keypoints] = prediction;
handImage.dispose();
const confidenceVal = confidence.dataSync()[0];
confidence.dispose();
if (confidenceVal < config.minConfidence) {
keypoints.dispose();
this.regionsOfInterest[i] = [];
return hands;
}
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateRegionsOfInterest(nextBoundingBox, false /* force replace */, i);
const result = {
landmarks: coords,
confidence: confidenceVal,
box: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint,
},
};
hands.push(result);
}
return hands;
}
// eslint-disable-next-line class-methods-use-this
calculateLandmarksBoundingBox(landmarks) {
const xs = landmarks.map((d) => d[0]);
const ys = landmarks.map((d) => d[1]);
const startPoint = [Math.min(...xs), Math.min(...ys)];
const endPoint = [Math.max(...xs), Math.max(...ys)];
return { startPoint, endPoint };
}
// Updates regions of interest if the intersection over union between
// the incoming and previous regions falls below a threshold.
updateRegionsOfInterest(box, forceUpdate, index) {
if (forceUpdate) {
this.regionsOfInterest[index] = [box];
} else {
const previousBox = this.regionsOfInterest[index][0];
let iou = 0;
if (previousBox != null && previousBox.startPoint != null) {
const [boxStartX, boxStartY] = box.startPoint;
const [boxEndX, boxEndY] = box.endPoint;
const [previousBoxStartX, previousBoxStartY] = previousBox.startPoint;
const [previousBoxEndX, previousBoxEndY] = previousBox.endPoint;
const xStartMax = Math.max(boxStartX, previousBoxStartX);
const yStartMax = Math.max(boxStartY, previousBoxStartY);
const xEndMin = Math.min(boxEndX, previousBoxEndX);
const yEndMin = Math.min(boxEndY, previousBoxEndY);
const intersection = (xEndMin - xStartMax) * (yEndMin - yStartMax);
const boxArea = (boxEndX - boxStartX) * (boxEndY - boxStartY);
const previousBoxArea = (previousBoxEndX - previousBoxStartX) * (previousBoxEndY - boxStartY);
iou = intersection / (boxArea + previousBoxArea - intersection);
}
this.regionsOfInterest[index][0] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : box;
}
}
shouldUpdateRegionsOfInterest() {
return !this.regionsOfInterest || (this.regionsOfInterest.length === 0) || (this.runsWithoutHandDetector >= this.skipFrames);
}
}
exports.HandPipeline = HandPipeline;

40
src/handpose/util.js
View File

@ -1,15 +1,27 @@
/**
* @license
* Copyright 2020 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================================
*/
function normalizeRadians(angle) { function normalizeRadians(angle) {
return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI)); return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI));
} }
exports.normalizeRadians = normalizeRadians;
function computeRotation(point1, point2) { function computeRotation(point1, point2) {
const radians = Math.PI / 2 - Math.atan2(-(point2[1] - point1[1]), point2[0] - point1[0]); const radians = Math.PI / 2 - Math.atan2(-(point2[1] - point1[1]), point2[0] - point1[0]);
return normalizeRadians(radians); return normalizeRadians(radians);
} }
exports.computeRotation = computeRotation; const buildTranslationMatrix = (x, y) => [[1, 0, x], [0, 1, y], [0, 0, 1]];
const buildTranslationMatrix = (x, y) => ([[1, 0, x], [0, 1, y], [0, 0, 1]]);
function dot(v1, v2) { function dot(v1, v2) {
let product = 0; let product = 0;
for (let i = 0; i < v1.length; i++) { for (let i = 0; i < v1.length; i++) {
@ -17,8 +29,6 @@ function dot(v1, v2) {
} }
return product; return product;
} }
exports.dot = dot;
function getColumnFrom2DArr(arr, columnIndex) { function getColumnFrom2DArr(arr, columnIndex) {
const column = []; const column = [];
for (let i = 0; i < arr.length; i++) { for (let i = 0; i < arr.length; i++) {
@ -26,8 +36,6 @@ function getColumnFrom2DArr(arr, columnIndex) {
} }
return column; return column;
} }
exports.getColumnFrom2DArr = getColumnFrom2DArr;
function multiplyTransformMatrices(mat1, mat2) { function multiplyTransformMatrices(mat1, mat2) {
const product = []; const product = [];
const size = mat1.length; const size = mat1.length;
@ -48,8 +56,6 @@ function buildRotationMatrix(rotation, center) {
const negativeTranslationMatrix = buildTranslationMatrix(-center[0], -center[1]); const negativeTranslationMatrix = buildTranslationMatrix(-center[0], -center[1]);
return multiplyTransformMatrices(translationTimesRotation, negativeTranslationMatrix); return multiplyTransformMatrices(translationTimesRotation, negativeTranslationMatrix);
} }
exports.buildRotationMatrix = buildRotationMatrix;
function invertTransformMatrix(matrix) { function invertTransformMatrix(matrix) {
const rotationComponent = [[matrix[0][0], matrix[1][0]], [matrix[0][1], matrix[1][1]]]; const rotationComponent = [[matrix[0][0], matrix[1][0]], [matrix[0][1], matrix[1][1]]];
const translationComponent = [matrix[0][2], matrix[1][2]]; const translationComponent = [matrix[0][2], matrix[1][2]];
@ -63,12 +69,18 @@ function invertTransformMatrix(matrix) {
[0, 0, 1], [0, 0, 1],
]; ];
} }
exports.invertTransformMatrix = invertTransformMatrix;
function rotatePoint(homogeneousCoordinate, rotationMatrix) { function rotatePoint(homogeneousCoordinate, rotationMatrix) {
return [ return [
dot(homogeneousCoordinate, rotationMatrix[0]), dot(homogeneousCoordinate, rotationMatrix[0]),
dot(homogeneousCoordinate, rotationMatrix[1]), dot(homogeneousCoordinate, rotationMatrix[1]),
]; ];
} }
exports.rotatePoint = rotatePoint; export {
buildRotationMatrix,
computeRotation,
dot,
getColumnFrom2DArr,
invertTransformMatrix,
normalizeRadians,
rotatePoint,
};

53
src/human.js
View File

@ -167,6 +167,7 @@ class Human {
this.log('Changing WebGL: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate); this.log('Changing WebGL: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1); tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
} }
tf.ENV.set('WEBGL_CPU_FORWARD', true);
await tf.ready(); await tf.ready();
} }
} }
@ -284,32 +285,6 @@ class Human {
perf.image = Math.trunc(now() - timeStamp); perf.image = Math.trunc(now() - timeStamp);
const imageTensor = image.tensor; const imageTensor = image.tensor;
// run posenet
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = 'run:body';
timeStamp = now();
this.analyze('Start PoseNet');
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze('End PoseNet:');
perf.body = Math.trunc(now() - timeStamp);
}
// run handpose
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = 'run:hand';
timeStamp = now();
this.analyze('Start HandPose:');
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze('End HandPose:');
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async) [poseRes, handRes] = await Promise.all([poseRes, handRes]);
// run facemesh, includes blazeface and iris // run facemesh, includes blazeface and iris
const faceRes = []; const faceRes = [];
if (this.config.face.enabled) { if (this.config.face.enabled) {
@ -357,6 +332,32 @@ class Human {
} }
} }
// run posenet
if (this.config.async) {
poseRes = this.config.body.enabled ? this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
} else {
this.state = 'run:body';
timeStamp = now();
this.analyze('Start PoseNet');
poseRes = this.config.body.enabled ? await this.models.posenet.estimatePoses(imageTensor, this.config.body) : [];
this.analyze('End PoseNet:');
perf.body = Math.trunc(now() - timeStamp);
}
// run handpose
if (this.config.async) {
handRes = this.config.hand.enabled ? this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
} else {
this.state = 'run:hand';
timeStamp = now();
this.analyze('Start HandPose:');
handRes = this.config.hand.enabled ? await this.models.handpose.estimateHands(imageTensor, this.config.hand) : [];
this.analyze('End HandPose:');
perf.hand = Math.trunc(now() - timeStamp);
}
if (this.config.async) [poseRes, handRes] = await Promise.all([poseRes, handRes]);
imageTensor.dispose(); imageTensor.dispose();
this.state = 'idle'; this.state = 'idle';