mirror of https://github.com/vladmandic/human
implement finger poses in hand detection and gestures
Vladimir Mandic
parent
348176b180
commit
a6ea8f5ca3
|
|
@ -9,7 +9,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/08/19 mandic00@live.com
|
||||
### **HEAD -> main** 2021/08/20 mandic00@live.com
|
||||
|
||||
|
||||
### **2.1.4** 2021/08/19 mandic00@live.com
|
||||
|
|
|
|||
|
|
@ -31,23 +31,26 @@
|
|||
"keypoints": [{"score":0.7,"part":"nose","positionRaw":[0.5763444304466248,0.11134015768766403],"position":[576,145]},{"score":0.61,"part":"leftEye","positionRaw":[0.5944706201553345,0.10566485673189163],"position":[594,137]},{"score":0.82,"part":"rightEye","positionRaw":[0.5578470230102539,0.09248857945203781],"position":[558,120]},{"score":0.23,"part":"leftEar","positionRaw":[0.6052790880203247,0.12891994416713715],"position":[605,168]},{"score":0.41,"part":"rightEar","positionRaw":[0.5136336088180542,0.10578916221857071],"position":[514,138]},{"score":0.54,"part":"leftShoulder","positionRaw":[0.5964832305908203,0.21158094704151154],"position":[596,275]},{"score":0.61,"part":"rightShoulder","positionRaw":[0.46281856298446655,0.17411141097545624],"position":[463,226]},{"score":0.5,"part":"leftElbow","positionRaw":[0.6168122291564941,0.3124275207519531],"position":[617,406]},{"score":0.66,"part":"rightElbow","positionRaw":[0.3865360915660858,0.07774468511343002],"position":[387,101]},{"score":0.47,"part":"leftWrist","positionRaw":[0.6024561524391174,0.21729949116706848],"position":[602,282]},{"score":0.24,"part":"rightWrist","positionRaw":[0.460940420627594,0.10492551326751709],"position":[461,136]},{"score":0.65,"part":"leftHip","positionRaw":[0.5450115203857422,0.4253421127796173],"position":[545,553]},{"score":0.64,"part":"rightHip","positionRaw":[0.43963193893432617,0.42362260818481445],"position":[440,551]},{"score":0.5,"part":"leftKnee","positionRaw":[0.5384384989738464,0.6577324271202087],"position":[538,855]},{"score":0.61,"part":"rightKnee","positionRaw":[0.5079054832458496,0.6598895192146301],"position":[508,858]},{"score":0.46,"part":"leftAnkle","positionRaw":[0.5350232720375061,0.8530999422073364],"position":[535,1109]},{"score":0.51,"part":"rightAnkle","positionRaw":[0.5480659008026123,0.8436547517776489],"position":[548,1097]}]
|
||||
}
|
||||
],
|
||||
|
||||
"hand":
|
||||
[
|
||||
{
|
||||
"id": 0,
|
||||
"score": 0.89,
|
||||
"box": [553,215,86,66],
|
||||
"boxRaw": [0.553,0.16538461538461538,0.086,0.05076923076923077],
|
||||
"keypoints": [[639,281,0],[608,278,12],[587,269,17],[569,268,17],[553,270,17],[598,224,17],[574,219,19],[561,227,18],[554,236,16],[603,219,9],[577,215,11],[564,223,11],[557,232,10],[606,222,2],[582,216,3],[569,223,4],[564,232,4],[607,230,-4],[587,225,-2],[577,228,-2],[571,233,-1]],
|
||||
"annotations": {"thumb":[[608,278,12],[587,269,17],[569,268,17],[553,270,17]],"indexFinger":[[598,224,17],[574,219,19],[561,227,18],[554,236,16]],"middleFinger":[[603,219,9],[577,215,11],[564,223,11],[557,232,10]],"ringFinger":[[606,222,2],[582,216,3],[569,223,4],[564,232,4]],"pinky":[[607,230,-4],[587,225,-2],[577,228,-2],[571,233,-1]],"palmBase":[[639,281,0]]}
|
||||
"id": 0,
|
||||
"score": 0.97,
|
||||
"box": [215,152,252,439],
|
||||
"boxRaw": [0.16796875,0.2111111111111111,0.196875,0.6097222222222223],
|
||||
"keypoints": [[296,530,0],[350,544,-68],[392,545,-114],[427,563,-147],[467,591,-184],[341,381,-113],[353,292,-143],[377,236,-155],[397,202,-162],[291,346,-91],[306,239,-123],[331,185,-136],[355,152,-144],[252,332,-70],[258,240,-94],[276,193,-105],[294,162,-111],[221,332,-52],[215,262,-70],[219,224,-78],[230,195,-82]],
|
||||
"annotations": {"thumb":[[350,544,-68],[392,545,-114],[427,563,-147],[467,591,-184]],"index":[[341,381,-113],[353,292,-143],[377,236,-155],[397,202,-162]],"middle":[[291,346,-91],[306,239,-123],[331,185,-136],[355,152,-144]],"ring":[[252,332,-70],[258,240,-94],[276,193,-105],[294,162,-111]],"pinky":[[221,332,-52],[215,262,-70],[219,224,-78],[230,195,-82]],"palm":[[296,530,0]]},
|
||||
"landmarks": {"thumb":{"curl":"none","direction":"horizontalRight"},"index":{"curl":"none","direction":"diagonalUpRight"},"middle":{"curl":"none","direction":"verticalUp"},"ring":{"curl":"none","direction":"verticalUp"},"pinky":{"curl":"none","direction":"verticalUp"}}
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"score": 0.97,
|
||||
"box": [539,213,84,113],
|
||||
"boxRaw": [0.539,0.16384615384615384,0.084,0.08692307692307692],
|
||||
"keypoints": [[623,326,0],[594,297,16],[579,272,21],[561,255,22],[544,241,23],[608,241,13],[582,216,12],[558,215,11],[543,221,10],[610,240,2],[579,213,0],[552,217,-3],[539,227,-3],[608,244,-7],[580,220,-10],[556,223,-11],[545,233,-11],[602,252,-16],[579,234,-18],[561,234,-18],[551,240,-17]],
|
||||
"annotations": {"thumb":[[594,297,16],[579,272,21],[561,255,22],[544,241,23]],"indexFinger":[[608,241,13],[582,216,12],[558,215,11],[543,221,10]],"middleFinger":[[610,240,2],[579,213,0],[552,217,-3],[539,227,-3]],"ringFinger":[[608,244,-7],[580,220,-10],[556,223,-11],[545,233,-11]],"pinky":[[602,252,-16],[579,234,-18],[561,234,-18],[551,240,-17]],"palmBase":[[623,326,0]]}
|
||||
"id": 1,
|
||||
"score": 0.18,
|
||||
"box": [611,209,99,280],
|
||||
"boxRaw": [0.47734375,0.2902777777777778,0.07734375,0.3888888888888889],
|
||||
"keypoints": [[674,209,0],[647,284,-11],[645,347,-15],[642,399,-16],[642,431,-16],[700,404,-11],[677,482,-14],[639,489,-15],[611,471,-15],[709,409,-6],[684,483,-8],[643,484,-8],[616,462,-8],[710,404,-3],[687,468,-4],[651,468,-3],[628,449,-2],[707,393,-1],[690,439,-3],[662,446,-4],[642,437,-4]],
|
||||
"annotations": {"thumb":[[647,284,-11],[645,347,-15],[642,399,-16],[642,431,-16]],"index":[[700,404,-11],[677,482,-14],[639,489,-15],[611,471,-15]],"middle":[[709,409,-6],[684,483,-8],[643,484,-8],[616,462,-8]],"ring":[[710,404,-3],[687,468,-4],[651,468,-3],[628,449,-2]],"pinky":[[707,393,-1],[690,439,-3],[662,446,-4],[642,437,-4]],"palm":[[674,209,0]]},
|
||||
"landmarks": {"thumb":{"curl":"none","direction":"verticalDown"},"index":{"curl":"half","direction":"verticalDown"},"middle":{"curl":"half","direction":"verticalDown"},"ring":{"curl":"half","direction":"verticalDown"},"pinky":{"curl":"half","direction":"verticalDown"}}
|
||||
}
|
||||
],
|
||||
"gesture":
|
||||
|
|
@ -62,7 +65,7 @@
|
|||
},
|
||||
{
|
||||
"hand": 0,
|
||||
"gesture": "pinky forward middlefinger up"
|
||||
"gesture": "thumbs up"
|
||||
},
|
||||
{
|
||||
"hand": 1,
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ let userConfig = {
|
|||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: true,
|
||||
face: { enabled: false,
|
||||
detector: { return: false, rotation: true },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
|
|
@ -48,7 +48,7 @@ let userConfig = {
|
|||
emotion: { enabled: false },
|
||||
},
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
hand: { enabled: false },
|
||||
body: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'movenet-multipose.json' },
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4595,14 +4595,14 @@ function scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, sco
|
|||
}
|
||||
return localMaximum;
|
||||
}
|
||||
function buildPartWithScoreQueue(minConfidence, scores) {
|
||||
function buildPartWithScoreQueue(minConfidence2, scores) {
|
||||
const [height, width, numKeypoints] = scores.shape;
|
||||
const queue = new MaxHeap(height * width * numKeypoints, ({ score: score3 }) => score3);
|
||||
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
|
||||
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
|
||||
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
|
||||
const score3 = scores.get(heatmapY, heatmapX, keypointId);
|
||||
if (score3 < minConfidence)
|
||||
if (score3 < minConfidence2)
|
||||
continue;
|
||||
if (scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, scores))
|
||||
queue.enqueue({ score: score3, part: { heatmapY, heatmapX, id: keypointId } });
|
||||
|
|
@ -4628,19 +4628,19 @@ function getInstanceScore(existingPoses, keypoints3) {
|
|||
}, 0);
|
||||
return notOverlappedKeypointScores / keypoints3.length;
|
||||
}
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) {
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence2) {
|
||||
const poses2 = [];
|
||||
const queue = buildPartWithScoreQueue(minConfidence, scores);
|
||||
const queue = buildPartWithScoreQueue(minConfidence2, scores);
|
||||
while (poses2.length < maxDetected && !queue.empty()) {
|
||||
const root = queue.dequeue();
|
||||
const rootImageCoords = getImageCoords(root.part, outputStride, offsets);
|
||||
if (withinRadius(poses2, rootImageCoords, root.part.id))
|
||||
continue;
|
||||
let keypoints3 = decodePose(root, scores, offsets, displacementsFwd, displacementsBwd);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence2);
|
||||
const score3 = getInstanceScore(poses2, keypoints3);
|
||||
const box6 = getBoundingBox(keypoints3);
|
||||
if (score3 > minConfidence)
|
||||
if (score3 > minConfidence2)
|
||||
poses2.push({ keypoints: keypoints3, box: box6, score: Math.round(100 * score3) / 100 });
|
||||
}
|
||||
return poses2;
|
||||
|
|
@ -7959,14 +7959,383 @@ var HandPipeline = class {
|
|||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/description.ts
|
||||
var Finger = {
|
||||
thumb: 0,
|
||||
index: 1,
|
||||
middle: 2,
|
||||
ring: 3,
|
||||
pinky: 4,
|
||||
all: [0, 1, 2, 3, 4],
|
||||
nameMapping: { 0: "thumb", 1: "index", 2: "middle", 3: "ring", 4: "pinky" },
|
||||
pointsMapping: {
|
||||
0: [[0, 1], [1, 2], [2, 3], [3, 4]],
|
||||
1: [[0, 5], [5, 6], [6, 7], [7, 8]],
|
||||
2: [[0, 9], [9, 10], [10, 11], [11, 12]],
|
||||
3: [[0, 13], [13, 14], [14, 15], [15, 16]],
|
||||
4: [[0, 17], [17, 18], [18, 19], [19, 20]]
|
||||
},
|
||||
getName: (value) => Finger.nameMapping[value],
|
||||
getPoints: (value) => Finger.pointsMapping[value]
|
||||
};
|
||||
var FingerCurl = {
|
||||
none: 0,
|
||||
half: 1,
|
||||
full: 2,
|
||||
nameMapping: { 0: "none", 1: "half", 2: "full" },
|
||||
getName: (value) => FingerCurl.nameMapping[value]
|
||||
};
|
||||
var FingerDirection = {
|
||||
verticalUp: 0,
|
||||
verticalDown: 1,
|
||||
horizontalLeft: 2,
|
||||
horizontalRight: 3,
|
||||
diagonalUpRight: 4,
|
||||
diagonalUpLeft: 5,
|
||||
diagonalDownRight: 6,
|
||||
diagonalDownLeft: 7,
|
||||
nameMapping: { 0: "verticalUp", 1: "verticalDown", 2: "horizontalLeft", 3: "horizontalRight", 4: "diagonalUpRight", 5: "diagonalUpLeft", 6: "diagonalDownRight", 7: "diagonalDownLeft" },
|
||||
getName: (value) => FingerDirection.nameMapping[value]
|
||||
};
|
||||
|
||||
// src/fingerpose/estimator.ts
|
||||
var options = {
|
||||
HALF_CURL_START_LIMIT: 60,
|
||||
NO_CURL_START_LIMIT: 130,
|
||||
DISTANCE_VOTE_POWER: 1.1,
|
||||
SINGLE_ANGLE_VOTE_POWER: 0.9,
|
||||
TOTAL_ANGLE_VOTE_POWER: 1.6
|
||||
};
|
||||
function calculateSlope(point1x, point1y, point2x, point2y) {
|
||||
const value = (point1y - point2y) / (point1x - point2x);
|
||||
let slope = Math.atan(value) * 180 / Math.PI;
|
||||
if (slope <= 0)
|
||||
slope = -slope;
|
||||
else if (slope > 0)
|
||||
slope = 180 - slope;
|
||||
return slope;
|
||||
}
|
||||
function getSlopes(point1, point2) {
|
||||
const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
|
||||
if (point1.length === 2)
|
||||
return slopeXY;
|
||||
const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
|
||||
return [slopeXY, slopeYZ];
|
||||
}
|
||||
function angleOrientationAt(angle, weightageAt = 1) {
|
||||
let isVertical = 0;
|
||||
let isDiagonal = 0;
|
||||
let isHorizontal = 0;
|
||||
if (angle >= 75 && angle <= 105)
|
||||
isVertical = 1 * weightageAt;
|
||||
else if (angle >= 25 && angle <= 155)
|
||||
isDiagonal = 1 * weightageAt;
|
||||
else
|
||||
isHorizontal = 1 * weightageAt;
|
||||
return [isVertical, isDiagonal, isHorizontal];
|
||||
}
|
||||
function estimateFingerCurl(startPoint, midPoint, endPoint) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const start_mid_z_dist = startPoint[2] - midPoint[2];
|
||||
const start_end_z_dist = startPoint[2] - endPoint[2];
|
||||
const mid_end_z_dist = midPoint[2] - endPoint[2];
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
|
||||
let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
|
||||
if (cos_in > 1)
|
||||
cos_in = 1;
|
||||
else if (cos_in < -1)
|
||||
cos_in = -1;
|
||||
let angleOfCurve = Math.acos(cos_in);
|
||||
angleOfCurve = 57.2958 * angleOfCurve % 180;
|
||||
let fingerCurl;
|
||||
if (angleOfCurve > options.NO_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.none;
|
||||
else if (angleOfCurve > options.HALF_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.half;
|
||||
else
|
||||
fingerCurl = FingerCurl.full;
|
||||
return fingerCurl;
|
||||
}
|
||||
function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_x === Math.abs(start_end_x_dist)) {
|
||||
if (start_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else if (max_dist_x === Math.abs(start_mid_x_dist)) {
|
||||
if (start_mid_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else {
|
||||
if (mid_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_y === Math.abs(start_end_y_dist)) {
|
||||
if (start_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else if (max_dist_y === Math.abs(start_mid_y_dist)) {
|
||||
if (start_mid_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else {
|
||||
if (mid_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
if (reqd_vertical_direction === FingerDirection.verticalUp) {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalUpLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalUpRight;
|
||||
} else {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalDownLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalDownRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
|
||||
const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
|
||||
let voteVertical = 0;
|
||||
let voteDiagonal = 0;
|
||||
let voteHorizontal = 0;
|
||||
const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 1e-5);
|
||||
if (start_end_x_y_dist_ratio > 1.5)
|
||||
voteVertical += options.DISTANCE_VOTE_POWER;
|
||||
else if (start_end_x_y_dist_ratio > 0.66)
|
||||
voteDiagonal += options.DISTANCE_VOTE_POWER;
|
||||
else
|
||||
voteHorizontal += options.DISTANCE_VOTE_POWER;
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
|
||||
const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
|
||||
let calc_start_point_x = startPoint[0];
|
||||
let calc_start_point_y = startPoint[1];
|
||||
let calc_end_point_x = endPoint[0];
|
||||
let calc_end_point_y = endPoint[1];
|
||||
if (max_dist === start_mid_dist) {
|
||||
calc_end_point_x = endPoint[0];
|
||||
calc_end_point_y = endPoint[1];
|
||||
} else if (max_dist === mid_end_dist) {
|
||||
calc_start_point_x = midPoint[0];
|
||||
calc_start_point_y = midPoint[1];
|
||||
}
|
||||
const calcStartPoint = [calc_start_point_x, calc_start_point_y];
|
||||
const calcEndPoint = [calc_end_point_x, calc_end_point_y];
|
||||
const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
|
||||
const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
|
||||
voteVertical += votes[0];
|
||||
voteDiagonal += votes[1];
|
||||
voteHorizontal += votes[2];
|
||||
for (const fingerSlope of fingerSlopes) {
|
||||
const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
|
||||
voteVertical += fingerVotes[0];
|
||||
voteDiagonal += fingerVotes[1];
|
||||
voteHorizontal += fingerVotes[2];
|
||||
}
|
||||
let estimatedDirection;
|
||||
if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
} else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
} else {
|
||||
estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
const slopeAtYZ = [];
|
||||
for (const point2 of points) {
|
||||
const point1 = landmarks[point2[0]];
|
||||
const point22 = landmarks[point2[1]];
|
||||
const slopes = getSlopes(point1, point22);
|
||||
const slopeXY = slopes[0];
|
||||
const slopeYZ = slopes[1];
|
||||
slopeAtXY.push(slopeXY);
|
||||
slopeAtYZ.push(slopeYZ);
|
||||
}
|
||||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
|
||||
const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
|
||||
const endPoint = landmarks[fingerPointsAt[3][1]];
|
||||
const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
|
||||
const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
|
||||
fingerCurls[finger] = fingerCurled;
|
||||
fingerDirections[finger] = fingerPosition;
|
||||
}
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
}
|
||||
|
||||
// src/fingerpose/gesture.ts
|
||||
var Gesture = class {
|
||||
constructor(name) {
|
||||
this.name = name;
|
||||
this.curls = {};
|
||||
this.directions = {};
|
||||
this.weights = [1, 1, 1, 1, 1];
|
||||
this.weightsRelative = [1, 1, 1, 1, 1];
|
||||
}
|
||||
addCurl(finger, curl, confidence) {
|
||||
if (typeof this.curls[finger] === "undefined")
|
||||
this.curls[finger] = [];
|
||||
this.curls[finger].push([curl, confidence]);
|
||||
}
|
||||
addDirection(finger, position, confidence) {
|
||||
if (!this.directions[finger])
|
||||
this.directions[finger] = [];
|
||||
this.directions[finger].push([position, confidence]);
|
||||
}
|
||||
setWeight(finger, weight) {
|
||||
this.weights[finger] = weight;
|
||||
const total = this.weights.reduce((a, b) => a + b, 0);
|
||||
this.weightsRelative = this.weights.map((el) => el * 5 / total);
|
||||
}
|
||||
matchAgainst(detectedCurls, detectedDirections) {
|
||||
let confidence = 0;
|
||||
for (const fingerIdx in detectedCurls) {
|
||||
const detectedCurl = detectedCurls[fingerIdx];
|
||||
const expectedCurls = this.curls[fingerIdx];
|
||||
if (typeof expectedCurls === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedCurl, score3] of expectedCurls) {
|
||||
if (detectedCurl === expectedCurl) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const fingerIdx in detectedDirections) {
|
||||
const detectedDirection = detectedDirections[fingerIdx];
|
||||
const expectedDirections = this.directions[fingerIdx];
|
||||
if (typeof expectedDirections === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedDirection, score3] of expectedDirections) {
|
||||
if (detectedDirection === expectedDirection) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return confidence / 10;
|
||||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/gestures.ts
|
||||
var ThumbsUp = new Gesture("thumbs up");
|
||||
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
|
||||
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
|
||||
ThumbsUp.addCurl(finger, FingerCurl.full, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1);
|
||||
}
|
||||
var Victory = new Gesture("victory");
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.index, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
|
||||
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.middle, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
|
||||
Victory.addCurl(Finger.ring, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.addCurl(Finger.pinky, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.setWeight(Finger.index, 2);
|
||||
Victory.setWeight(Finger.middle, 2);
|
||||
var gestures_default = [ThumbsUp, Victory];
|
||||
|
||||
// src/fingerpose/fingerpose.ts
|
||||
var minConfidence = 0.7;
|
||||
function analyze(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const landmarks = {};
|
||||
for (const fingerIdx of Finger.all) {
|
||||
landmarks[Finger.getName(fingerIdx)] = {
|
||||
curl: FingerCurl.getName(estimatorRes.curls[fingerIdx]),
|
||||
direction: FingerDirection.getName(estimatorRes.directions[fingerIdx])
|
||||
};
|
||||
}
|
||||
return landmarks;
|
||||
}
|
||||
function match2(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const poses2 = [];
|
||||
for (const gesture3 of gestures_default) {
|
||||
const confidence = gesture3.matchAgainst(estimatorRes.curls, estimatorRes.directions);
|
||||
if (confidence >= minConfidence)
|
||||
poses2.push({ name: gesture3.name, confidence });
|
||||
}
|
||||
return poses2;
|
||||
}
|
||||
|
||||
// src/handpose/handpose.ts
|
||||
var meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
indexFinger: [5, 6, 7, 8],
|
||||
middleFinger: [9, 10, 11, 12],
|
||||
ringFinger: [13, 14, 15, 16],
|
||||
index: [5, 6, 7, 8],
|
||||
middle: [9, 10, 11, 12],
|
||||
ring: [13, 14, 15, 16],
|
||||
pinky: [17, 18, 19, 20],
|
||||
palmBase: [0]
|
||||
palm: [0]
|
||||
};
|
||||
var handDetectorModel;
|
||||
var handPoseModel;
|
||||
|
|
@ -8014,7 +8383,16 @@ async function predict5(input, config3) {
|
|||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0)
|
||||
];
|
||||
}
|
||||
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box: box6, boxRaw: boxRaw3, keypoints: keypoints3, annotations: annotations3 });
|
||||
const landmarks = analyze(keypoints3);
|
||||
hands.push({
|
||||
id: i,
|
||||
score: Math.round(100 * predictions[i].confidence) / 100,
|
||||
box: box6,
|
||||
boxRaw: boxRaw3,
|
||||
keypoints: keypoints3,
|
||||
annotations: annotations3,
|
||||
landmarks
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
@ -8706,9 +9084,9 @@ var tf18 = __toModule(require_tfjs_esm());
|
|||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
const _collect = function(source, prefix, collection) {
|
||||
const r = new RegExp("\\b" + prefix + " \\w+ (\\w+)", "ig");
|
||||
source.replace(r, (match2, name) => {
|
||||
source.replace(r, (match3, name) => {
|
||||
collection[name] = 0;
|
||||
return match2;
|
||||
return match3;
|
||||
});
|
||||
};
|
||||
const _compile = function(source, type) {
|
||||
|
|
@ -10013,6 +10391,9 @@ var hand = (res) => {
|
|||
const highest = fingers.reduce((best, a) => best.position[1] < a.position[1] ? best : a);
|
||||
gestures.push({ hand: i, gesture: `${highest.name} up` });
|
||||
}
|
||||
const poses2 = match2(res[i]["keypoints"]);
|
||||
for (const pose of poses2)
|
||||
gestures.push({ hand: i, gesture: pose.name });
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
@ -10027,10 +10408,10 @@ __export(draw_exports, {
|
|||
gesture: () => gesture,
|
||||
hand: () => hand2,
|
||||
object: () => object,
|
||||
options: () => options,
|
||||
options: () => options2,
|
||||
person: () => person
|
||||
});
|
||||
var options = {
|
||||
var options2 = {
|
||||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
|
|
@ -10115,7 +10496,7 @@ function curves(ctx, points = [], localOptions) {
|
|||
}
|
||||
}
|
||||
async function gesture(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10145,7 +10526,7 @@ async function gesture(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function face2(inCanvas2, result, drawOptions) {
|
||||
var _a, _b, _c, _d;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10255,7 +10636,7 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function body2(inCanvas2, result, drawOptions) {
|
||||
var _a;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10387,7 +10768,7 @@ async function body2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function hand2(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10426,12 +10807,12 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations["indexFinger"], "index");
|
||||
addHandLabel(h.annotations["middleFinger"], "middle");
|
||||
addHandLabel(h.annotations["ringFinger"], "ring");
|
||||
addHandLabel(h.annotations["index"], "index");
|
||||
addHandLabel(h.annotations["middle"], "middle");
|
||||
addHandLabel(h.annotations["ring"], "ring");
|
||||
addHandLabel(h.annotations["pinky"], "pinky");
|
||||
addHandLabel(h.annotations["thumb"], "thumb");
|
||||
addHandLabel(h.annotations["palmBase"], "palm");
|
||||
addHandLabel(h.annotations["palm"], "palm");
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -10446,16 +10827,16 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations["indexFinger"]);
|
||||
addHandLine(h.annotations["middleFinger"]);
|
||||
addHandLine(h.annotations["ringFinger"]);
|
||||
addHandLine(h.annotations["index"]);
|
||||
addHandLine(h.annotations["middle"]);
|
||||
addHandLine(h.annotations["ring"]);
|
||||
addHandLine(h.annotations["pinky"]);
|
||||
addHandLine(h.annotations["thumb"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function object(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10484,7 +10865,7 @@ async function object(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function person(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10522,7 +10903,7 @@ async function canvas(inCanvas2, outCanvas2) {
|
|||
}
|
||||
async function all(inCanvas2, result, drawOptions) {
|
||||
const timestamp = now();
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return null;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
|
|||
|
|
@ -4596,14 +4596,14 @@ function scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, sco
|
|||
}
|
||||
return localMaximum;
|
||||
}
|
||||
function buildPartWithScoreQueue(minConfidence, scores) {
|
||||
function buildPartWithScoreQueue(minConfidence2, scores) {
|
||||
const [height, width, numKeypoints] = scores.shape;
|
||||
const queue = new MaxHeap(height * width * numKeypoints, ({ score: score3 }) => score3);
|
||||
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
|
||||
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
|
||||
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
|
||||
const score3 = scores.get(heatmapY, heatmapX, keypointId);
|
||||
if (score3 < minConfidence)
|
||||
if (score3 < minConfidence2)
|
||||
continue;
|
||||
if (scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, scores))
|
||||
queue.enqueue({ score: score3, part: { heatmapY, heatmapX, id: keypointId } });
|
||||
|
|
@ -4629,19 +4629,19 @@ function getInstanceScore(existingPoses, keypoints3) {
|
|||
}, 0);
|
||||
return notOverlappedKeypointScores / keypoints3.length;
|
||||
}
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) {
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence2) {
|
||||
const poses2 = [];
|
||||
const queue = buildPartWithScoreQueue(minConfidence, scores);
|
||||
const queue = buildPartWithScoreQueue(minConfidence2, scores);
|
||||
while (poses2.length < maxDetected && !queue.empty()) {
|
||||
const root = queue.dequeue();
|
||||
const rootImageCoords = getImageCoords(root.part, outputStride, offsets);
|
||||
if (withinRadius(poses2, rootImageCoords, root.part.id))
|
||||
continue;
|
||||
let keypoints3 = decodePose(root, scores, offsets, displacementsFwd, displacementsBwd);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence2);
|
||||
const score3 = getInstanceScore(poses2, keypoints3);
|
||||
const box6 = getBoundingBox(keypoints3);
|
||||
if (score3 > minConfidence)
|
||||
if (score3 > minConfidence2)
|
||||
poses2.push({ keypoints: keypoints3, box: box6, score: Math.round(100 * score3) / 100 });
|
||||
}
|
||||
return poses2;
|
||||
|
|
@ -7960,14 +7960,383 @@ var HandPipeline = class {
|
|||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/description.ts
|
||||
var Finger = {
|
||||
thumb: 0,
|
||||
index: 1,
|
||||
middle: 2,
|
||||
ring: 3,
|
||||
pinky: 4,
|
||||
all: [0, 1, 2, 3, 4],
|
||||
nameMapping: { 0: "thumb", 1: "index", 2: "middle", 3: "ring", 4: "pinky" },
|
||||
pointsMapping: {
|
||||
0: [[0, 1], [1, 2], [2, 3], [3, 4]],
|
||||
1: [[0, 5], [5, 6], [6, 7], [7, 8]],
|
||||
2: [[0, 9], [9, 10], [10, 11], [11, 12]],
|
||||
3: [[0, 13], [13, 14], [14, 15], [15, 16]],
|
||||
4: [[0, 17], [17, 18], [18, 19], [19, 20]]
|
||||
},
|
||||
getName: (value) => Finger.nameMapping[value],
|
||||
getPoints: (value) => Finger.pointsMapping[value]
|
||||
};
|
||||
var FingerCurl = {
|
||||
none: 0,
|
||||
half: 1,
|
||||
full: 2,
|
||||
nameMapping: { 0: "none", 1: "half", 2: "full" },
|
||||
getName: (value) => FingerCurl.nameMapping[value]
|
||||
};
|
||||
var FingerDirection = {
|
||||
verticalUp: 0,
|
||||
verticalDown: 1,
|
||||
horizontalLeft: 2,
|
||||
horizontalRight: 3,
|
||||
diagonalUpRight: 4,
|
||||
diagonalUpLeft: 5,
|
||||
diagonalDownRight: 6,
|
||||
diagonalDownLeft: 7,
|
||||
nameMapping: { 0: "verticalUp", 1: "verticalDown", 2: "horizontalLeft", 3: "horizontalRight", 4: "diagonalUpRight", 5: "diagonalUpLeft", 6: "diagonalDownRight", 7: "diagonalDownLeft" },
|
||||
getName: (value) => FingerDirection.nameMapping[value]
|
||||
};
|
||||
|
||||
// src/fingerpose/estimator.ts
|
||||
var options = {
|
||||
HALF_CURL_START_LIMIT: 60,
|
||||
NO_CURL_START_LIMIT: 130,
|
||||
DISTANCE_VOTE_POWER: 1.1,
|
||||
SINGLE_ANGLE_VOTE_POWER: 0.9,
|
||||
TOTAL_ANGLE_VOTE_POWER: 1.6
|
||||
};
|
||||
function calculateSlope(point1x, point1y, point2x, point2y) {
|
||||
const value = (point1y - point2y) / (point1x - point2x);
|
||||
let slope = Math.atan(value) * 180 / Math.PI;
|
||||
if (slope <= 0)
|
||||
slope = -slope;
|
||||
else if (slope > 0)
|
||||
slope = 180 - slope;
|
||||
return slope;
|
||||
}
|
||||
function getSlopes(point1, point2) {
|
||||
const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
|
||||
if (point1.length === 2)
|
||||
return slopeXY;
|
||||
const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
|
||||
return [slopeXY, slopeYZ];
|
||||
}
|
||||
function angleOrientationAt(angle, weightageAt = 1) {
|
||||
let isVertical = 0;
|
||||
let isDiagonal = 0;
|
||||
let isHorizontal = 0;
|
||||
if (angle >= 75 && angle <= 105)
|
||||
isVertical = 1 * weightageAt;
|
||||
else if (angle >= 25 && angle <= 155)
|
||||
isDiagonal = 1 * weightageAt;
|
||||
else
|
||||
isHorizontal = 1 * weightageAt;
|
||||
return [isVertical, isDiagonal, isHorizontal];
|
||||
}
|
||||
function estimateFingerCurl(startPoint, midPoint, endPoint) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const start_mid_z_dist = startPoint[2] - midPoint[2];
|
||||
const start_end_z_dist = startPoint[2] - endPoint[2];
|
||||
const mid_end_z_dist = midPoint[2] - endPoint[2];
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
|
||||
let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
|
||||
if (cos_in > 1)
|
||||
cos_in = 1;
|
||||
else if (cos_in < -1)
|
||||
cos_in = -1;
|
||||
let angleOfCurve = Math.acos(cos_in);
|
||||
angleOfCurve = 57.2958 * angleOfCurve % 180;
|
||||
let fingerCurl;
|
||||
if (angleOfCurve > options.NO_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.none;
|
||||
else if (angleOfCurve > options.HALF_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.half;
|
||||
else
|
||||
fingerCurl = FingerCurl.full;
|
||||
return fingerCurl;
|
||||
}
|
||||
function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_x === Math.abs(start_end_x_dist)) {
|
||||
if (start_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else if (max_dist_x === Math.abs(start_mid_x_dist)) {
|
||||
if (start_mid_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else {
|
||||
if (mid_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_y === Math.abs(start_end_y_dist)) {
|
||||
if (start_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else if (max_dist_y === Math.abs(start_mid_y_dist)) {
|
||||
if (start_mid_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else {
|
||||
if (mid_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
if (reqd_vertical_direction === FingerDirection.verticalUp) {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalUpLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalUpRight;
|
||||
} else {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalDownLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalDownRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
|
||||
const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
|
||||
let voteVertical = 0;
|
||||
let voteDiagonal = 0;
|
||||
let voteHorizontal = 0;
|
||||
const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 1e-5);
|
||||
if (start_end_x_y_dist_ratio > 1.5)
|
||||
voteVertical += options.DISTANCE_VOTE_POWER;
|
||||
else if (start_end_x_y_dist_ratio > 0.66)
|
||||
voteDiagonal += options.DISTANCE_VOTE_POWER;
|
||||
else
|
||||
voteHorizontal += options.DISTANCE_VOTE_POWER;
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
|
||||
const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
|
||||
let calc_start_point_x = startPoint[0];
|
||||
let calc_start_point_y = startPoint[1];
|
||||
let calc_end_point_x = endPoint[0];
|
||||
let calc_end_point_y = endPoint[1];
|
||||
if (max_dist === start_mid_dist) {
|
||||
calc_end_point_x = endPoint[0];
|
||||
calc_end_point_y = endPoint[1];
|
||||
} else if (max_dist === mid_end_dist) {
|
||||
calc_start_point_x = midPoint[0];
|
||||
calc_start_point_y = midPoint[1];
|
||||
}
|
||||
const calcStartPoint = [calc_start_point_x, calc_start_point_y];
|
||||
const calcEndPoint = [calc_end_point_x, calc_end_point_y];
|
||||
const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
|
||||
const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
|
||||
voteVertical += votes[0];
|
||||
voteDiagonal += votes[1];
|
||||
voteHorizontal += votes[2];
|
||||
for (const fingerSlope of fingerSlopes) {
|
||||
const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
|
||||
voteVertical += fingerVotes[0];
|
||||
voteDiagonal += fingerVotes[1];
|
||||
voteHorizontal += fingerVotes[2];
|
||||
}
|
||||
let estimatedDirection;
|
||||
if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
} else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
} else {
|
||||
estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
const slopeAtYZ = [];
|
||||
for (const point2 of points) {
|
||||
const point1 = landmarks[point2[0]];
|
||||
const point22 = landmarks[point2[1]];
|
||||
const slopes = getSlopes(point1, point22);
|
||||
const slopeXY = slopes[0];
|
||||
const slopeYZ = slopes[1];
|
||||
slopeAtXY.push(slopeXY);
|
||||
slopeAtYZ.push(slopeYZ);
|
||||
}
|
||||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
|
||||
const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
|
||||
const endPoint = landmarks[fingerPointsAt[3][1]];
|
||||
const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
|
||||
const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
|
||||
fingerCurls[finger] = fingerCurled;
|
||||
fingerDirections[finger] = fingerPosition;
|
||||
}
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
}
|
||||
|
||||
// src/fingerpose/gesture.ts
|
||||
var Gesture = class {
|
||||
constructor(name) {
|
||||
this.name = name;
|
||||
this.curls = {};
|
||||
this.directions = {};
|
||||
this.weights = [1, 1, 1, 1, 1];
|
||||
this.weightsRelative = [1, 1, 1, 1, 1];
|
||||
}
|
||||
addCurl(finger, curl, confidence) {
|
||||
if (typeof this.curls[finger] === "undefined")
|
||||
this.curls[finger] = [];
|
||||
this.curls[finger].push([curl, confidence]);
|
||||
}
|
||||
addDirection(finger, position, confidence) {
|
||||
if (!this.directions[finger])
|
||||
this.directions[finger] = [];
|
||||
this.directions[finger].push([position, confidence]);
|
||||
}
|
||||
setWeight(finger, weight) {
|
||||
this.weights[finger] = weight;
|
||||
const total = this.weights.reduce((a, b) => a + b, 0);
|
||||
this.weightsRelative = this.weights.map((el) => el * 5 / total);
|
||||
}
|
||||
matchAgainst(detectedCurls, detectedDirections) {
|
||||
let confidence = 0;
|
||||
for (const fingerIdx in detectedCurls) {
|
||||
const detectedCurl = detectedCurls[fingerIdx];
|
||||
const expectedCurls = this.curls[fingerIdx];
|
||||
if (typeof expectedCurls === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedCurl, score3] of expectedCurls) {
|
||||
if (detectedCurl === expectedCurl) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const fingerIdx in detectedDirections) {
|
||||
const detectedDirection = detectedDirections[fingerIdx];
|
||||
const expectedDirections = this.directions[fingerIdx];
|
||||
if (typeof expectedDirections === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedDirection, score3] of expectedDirections) {
|
||||
if (detectedDirection === expectedDirection) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return confidence / 10;
|
||||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/gestures.ts
|
||||
var ThumbsUp = new Gesture("thumbs up");
|
||||
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
|
||||
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
|
||||
ThumbsUp.addCurl(finger, FingerCurl.full, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1);
|
||||
}
|
||||
var Victory = new Gesture("victory");
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.index, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
|
||||
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.middle, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
|
||||
Victory.addCurl(Finger.ring, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.addCurl(Finger.pinky, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.setWeight(Finger.index, 2);
|
||||
Victory.setWeight(Finger.middle, 2);
|
||||
var gestures_default = [ThumbsUp, Victory];
|
||||
|
||||
// src/fingerpose/fingerpose.ts
|
||||
var minConfidence = 0.7;
|
||||
function analyze(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const landmarks = {};
|
||||
for (const fingerIdx of Finger.all) {
|
||||
landmarks[Finger.getName(fingerIdx)] = {
|
||||
curl: FingerCurl.getName(estimatorRes.curls[fingerIdx]),
|
||||
direction: FingerDirection.getName(estimatorRes.directions[fingerIdx])
|
||||
};
|
||||
}
|
||||
return landmarks;
|
||||
}
|
||||
function match2(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const poses2 = [];
|
||||
for (const gesture3 of gestures_default) {
|
||||
const confidence = gesture3.matchAgainst(estimatorRes.curls, estimatorRes.directions);
|
||||
if (confidence >= minConfidence)
|
||||
poses2.push({ name: gesture3.name, confidence });
|
||||
}
|
||||
return poses2;
|
||||
}
|
||||
|
||||
// src/handpose/handpose.ts
|
||||
var meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
indexFinger: [5, 6, 7, 8],
|
||||
middleFinger: [9, 10, 11, 12],
|
||||
ringFinger: [13, 14, 15, 16],
|
||||
index: [5, 6, 7, 8],
|
||||
middle: [9, 10, 11, 12],
|
||||
ring: [13, 14, 15, 16],
|
||||
pinky: [17, 18, 19, 20],
|
||||
palmBase: [0]
|
||||
palm: [0]
|
||||
};
|
||||
var handDetectorModel;
|
||||
var handPoseModel;
|
||||
|
|
@ -8015,7 +8384,16 @@ async function predict5(input, config3) {
|
|||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0)
|
||||
];
|
||||
}
|
||||
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box: box6, boxRaw: boxRaw3, keypoints: keypoints3, annotations: annotations3 });
|
||||
const landmarks = analyze(keypoints3);
|
||||
hands.push({
|
||||
id: i,
|
||||
score: Math.round(100 * predictions[i].confidence) / 100,
|
||||
box: box6,
|
||||
boxRaw: boxRaw3,
|
||||
keypoints: keypoints3,
|
||||
annotations: annotations3,
|
||||
landmarks
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
@ -8707,9 +9085,9 @@ var tf18 = __toModule(require_tfjs_esm());
|
|||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
const _collect = function(source, prefix, collection) {
|
||||
const r = new RegExp("\\b" + prefix + " \\w+ (\\w+)", "ig");
|
||||
source.replace(r, (match2, name) => {
|
||||
source.replace(r, (match3, name) => {
|
||||
collection[name] = 0;
|
||||
return match2;
|
||||
return match3;
|
||||
});
|
||||
};
|
||||
const _compile = function(source, type) {
|
||||
|
|
@ -10014,6 +10392,9 @@ var hand = (res) => {
|
|||
const highest = fingers.reduce((best, a) => best.position[1] < a.position[1] ? best : a);
|
||||
gestures.push({ hand: i, gesture: `${highest.name} up` });
|
||||
}
|
||||
const poses2 = match2(res[i]["keypoints"]);
|
||||
for (const pose of poses2)
|
||||
gestures.push({ hand: i, gesture: pose.name });
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
@ -10028,10 +10409,10 @@ __export(draw_exports, {
|
|||
gesture: () => gesture,
|
||||
hand: () => hand2,
|
||||
object: () => object,
|
||||
options: () => options,
|
||||
options: () => options2,
|
||||
person: () => person
|
||||
});
|
||||
var options = {
|
||||
var options2 = {
|
||||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
|
|
@ -10116,7 +10497,7 @@ function curves(ctx, points = [], localOptions) {
|
|||
}
|
||||
}
|
||||
async function gesture(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10146,7 +10527,7 @@ async function gesture(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function face2(inCanvas2, result, drawOptions) {
|
||||
var _a, _b, _c, _d;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10256,7 +10637,7 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function body2(inCanvas2, result, drawOptions) {
|
||||
var _a;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10388,7 +10769,7 @@ async function body2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function hand2(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10427,12 +10808,12 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations["indexFinger"], "index");
|
||||
addHandLabel(h.annotations["middleFinger"], "middle");
|
||||
addHandLabel(h.annotations["ringFinger"], "ring");
|
||||
addHandLabel(h.annotations["index"], "index");
|
||||
addHandLabel(h.annotations["middle"], "middle");
|
||||
addHandLabel(h.annotations["ring"], "ring");
|
||||
addHandLabel(h.annotations["pinky"], "pinky");
|
||||
addHandLabel(h.annotations["thumb"], "thumb");
|
||||
addHandLabel(h.annotations["palmBase"], "palm");
|
||||
addHandLabel(h.annotations["palm"], "palm");
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -10447,16 +10828,16 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations["indexFinger"]);
|
||||
addHandLine(h.annotations["middleFinger"]);
|
||||
addHandLine(h.annotations["ringFinger"]);
|
||||
addHandLine(h.annotations["index"]);
|
||||
addHandLine(h.annotations["middle"]);
|
||||
addHandLine(h.annotations["ring"]);
|
||||
addHandLine(h.annotations["pinky"]);
|
||||
addHandLine(h.annotations["thumb"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function object(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10485,7 +10866,7 @@ async function object(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function person(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10523,7 +10904,7 @@ async function canvas(inCanvas2, outCanvas2) {
|
|||
}
|
||||
async function all(inCanvas2, result, drawOptions) {
|
||||
const timestamp = now();
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return null;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
|
|||
|
|
@ -4595,14 +4595,14 @@ function scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, sco
|
|||
}
|
||||
return localMaximum;
|
||||
}
|
||||
function buildPartWithScoreQueue(minConfidence, scores) {
|
||||
function buildPartWithScoreQueue(minConfidence2, scores) {
|
||||
const [height, width, numKeypoints] = scores.shape;
|
||||
const queue = new MaxHeap(height * width * numKeypoints, ({ score: score3 }) => score3);
|
||||
for (let heatmapY = 0; heatmapY < height; ++heatmapY) {
|
||||
for (let heatmapX = 0; heatmapX < width; ++heatmapX) {
|
||||
for (let keypointId = 0; keypointId < numKeypoints; ++keypointId) {
|
||||
const score3 = scores.get(heatmapY, heatmapX, keypointId);
|
||||
if (score3 < minConfidence)
|
||||
if (score3 < minConfidence2)
|
||||
continue;
|
||||
if (scoreIsMaximumInLocalWindow(keypointId, score3, heatmapY, heatmapX, scores))
|
||||
queue.enqueue({ score: score3, part: { heatmapY, heatmapX, id: keypointId } });
|
||||
|
|
@ -4628,19 +4628,19 @@ function getInstanceScore(existingPoses, keypoints3) {
|
|||
}, 0);
|
||||
return notOverlappedKeypointScores / keypoints3.length;
|
||||
}
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) {
|
||||
function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence2) {
|
||||
const poses2 = [];
|
||||
const queue = buildPartWithScoreQueue(minConfidence, scores);
|
||||
const queue = buildPartWithScoreQueue(minConfidence2, scores);
|
||||
while (poses2.length < maxDetected && !queue.empty()) {
|
||||
const root = queue.dequeue();
|
||||
const rootImageCoords = getImageCoords(root.part, outputStride, offsets);
|
||||
if (withinRadius(poses2, rootImageCoords, root.part.id))
|
||||
continue;
|
||||
let keypoints3 = decodePose(root, scores, offsets, displacementsFwd, displacementsBwd);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence);
|
||||
keypoints3 = keypoints3.filter((a) => a.score > minConfidence2);
|
||||
const score3 = getInstanceScore(poses2, keypoints3);
|
||||
const box6 = getBoundingBox(keypoints3);
|
||||
if (score3 > minConfidence)
|
||||
if (score3 > minConfidence2)
|
||||
poses2.push({ keypoints: keypoints3, box: box6, score: Math.round(100 * score3) / 100 });
|
||||
}
|
||||
return poses2;
|
||||
|
|
@ -7959,14 +7959,383 @@ var HandPipeline = class {
|
|||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/description.ts
|
||||
var Finger = {
|
||||
thumb: 0,
|
||||
index: 1,
|
||||
middle: 2,
|
||||
ring: 3,
|
||||
pinky: 4,
|
||||
all: [0, 1, 2, 3, 4],
|
||||
nameMapping: { 0: "thumb", 1: "index", 2: "middle", 3: "ring", 4: "pinky" },
|
||||
pointsMapping: {
|
||||
0: [[0, 1], [1, 2], [2, 3], [3, 4]],
|
||||
1: [[0, 5], [5, 6], [6, 7], [7, 8]],
|
||||
2: [[0, 9], [9, 10], [10, 11], [11, 12]],
|
||||
3: [[0, 13], [13, 14], [14, 15], [15, 16]],
|
||||
4: [[0, 17], [17, 18], [18, 19], [19, 20]]
|
||||
},
|
||||
getName: (value) => Finger.nameMapping[value],
|
||||
getPoints: (value) => Finger.pointsMapping[value]
|
||||
};
|
||||
var FingerCurl = {
|
||||
none: 0,
|
||||
half: 1,
|
||||
full: 2,
|
||||
nameMapping: { 0: "none", 1: "half", 2: "full" },
|
||||
getName: (value) => FingerCurl.nameMapping[value]
|
||||
};
|
||||
var FingerDirection = {
|
||||
verticalUp: 0,
|
||||
verticalDown: 1,
|
||||
horizontalLeft: 2,
|
||||
horizontalRight: 3,
|
||||
diagonalUpRight: 4,
|
||||
diagonalUpLeft: 5,
|
||||
diagonalDownRight: 6,
|
||||
diagonalDownLeft: 7,
|
||||
nameMapping: { 0: "verticalUp", 1: "verticalDown", 2: "horizontalLeft", 3: "horizontalRight", 4: "diagonalUpRight", 5: "diagonalUpLeft", 6: "diagonalDownRight", 7: "diagonalDownLeft" },
|
||||
getName: (value) => FingerDirection.nameMapping[value]
|
||||
};
|
||||
|
||||
// src/fingerpose/estimator.ts
|
||||
var options = {
|
||||
HALF_CURL_START_LIMIT: 60,
|
||||
NO_CURL_START_LIMIT: 130,
|
||||
DISTANCE_VOTE_POWER: 1.1,
|
||||
SINGLE_ANGLE_VOTE_POWER: 0.9,
|
||||
TOTAL_ANGLE_VOTE_POWER: 1.6
|
||||
};
|
||||
function calculateSlope(point1x, point1y, point2x, point2y) {
|
||||
const value = (point1y - point2y) / (point1x - point2x);
|
||||
let slope = Math.atan(value) * 180 / Math.PI;
|
||||
if (slope <= 0)
|
||||
slope = -slope;
|
||||
else if (slope > 0)
|
||||
slope = 180 - slope;
|
||||
return slope;
|
||||
}
|
||||
function getSlopes(point1, point2) {
|
||||
const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
|
||||
if (point1.length === 2)
|
||||
return slopeXY;
|
||||
const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
|
||||
return [slopeXY, slopeYZ];
|
||||
}
|
||||
function angleOrientationAt(angle, weightageAt = 1) {
|
||||
let isVertical = 0;
|
||||
let isDiagonal = 0;
|
||||
let isHorizontal = 0;
|
||||
if (angle >= 75 && angle <= 105)
|
||||
isVertical = 1 * weightageAt;
|
||||
else if (angle >= 25 && angle <= 155)
|
||||
isDiagonal = 1 * weightageAt;
|
||||
else
|
||||
isHorizontal = 1 * weightageAt;
|
||||
return [isVertical, isDiagonal, isHorizontal];
|
||||
}
|
||||
function estimateFingerCurl(startPoint, midPoint, endPoint) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const start_mid_z_dist = startPoint[2] - midPoint[2];
|
||||
const start_end_z_dist = startPoint[2] - endPoint[2];
|
||||
const mid_end_z_dist = midPoint[2] - endPoint[2];
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
|
||||
let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
|
||||
if (cos_in > 1)
|
||||
cos_in = 1;
|
||||
else if (cos_in < -1)
|
||||
cos_in = -1;
|
||||
let angleOfCurve = Math.acos(cos_in);
|
||||
angleOfCurve = 57.2958 * angleOfCurve % 180;
|
||||
let fingerCurl;
|
||||
if (angleOfCurve > options.NO_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.none;
|
||||
else if (angleOfCurve > options.HALF_CURL_START_LIMIT)
|
||||
fingerCurl = FingerCurl.half;
|
||||
else
|
||||
fingerCurl = FingerCurl.full;
|
||||
return fingerCurl;
|
||||
}
|
||||
function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_x === Math.abs(start_end_x_dist)) {
|
||||
if (start_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else if (max_dist_x === Math.abs(start_mid_x_dist)) {
|
||||
if (start_mid_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else {
|
||||
if (mid_end_x_dist > 0)
|
||||
estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.horizontalRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_y === Math.abs(start_end_y_dist)) {
|
||||
if (start_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else if (max_dist_y === Math.abs(start_mid_y_dist)) {
|
||||
if (start_mid_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
} else {
|
||||
if (mid_end_y_dist < 0)
|
||||
estimatedDirection = FingerDirection.verticalDown;
|
||||
else
|
||||
estimatedDirection = FingerDirection.verticalUp;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
if (reqd_vertical_direction === FingerDirection.verticalUp) {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalUpLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalUpRight;
|
||||
} else {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft)
|
||||
estimatedDirection = FingerDirection.diagonalDownLeft;
|
||||
else
|
||||
estimatedDirection = FingerDirection.diagonalDownRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
|
||||
const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
|
||||
let voteVertical = 0;
|
||||
let voteDiagonal = 0;
|
||||
let voteHorizontal = 0;
|
||||
const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 1e-5);
|
||||
if (start_end_x_y_dist_ratio > 1.5)
|
||||
voteVertical += options.DISTANCE_VOTE_POWER;
|
||||
else if (start_end_x_y_dist_ratio > 0.66)
|
||||
voteDiagonal += options.DISTANCE_VOTE_POWER;
|
||||
else
|
||||
voteHorizontal += options.DISTANCE_VOTE_POWER;
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
|
||||
const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
|
||||
let calc_start_point_x = startPoint[0];
|
||||
let calc_start_point_y = startPoint[1];
|
||||
let calc_end_point_x = endPoint[0];
|
||||
let calc_end_point_y = endPoint[1];
|
||||
if (max_dist === start_mid_dist) {
|
||||
calc_end_point_x = endPoint[0];
|
||||
calc_end_point_y = endPoint[1];
|
||||
} else if (max_dist === mid_end_dist) {
|
||||
calc_start_point_x = midPoint[0];
|
||||
calc_start_point_y = midPoint[1];
|
||||
}
|
||||
const calcStartPoint = [calc_start_point_x, calc_start_point_y];
|
||||
const calcEndPoint = [calc_end_point_x, calc_end_point_y];
|
||||
const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
|
||||
const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
|
||||
voteVertical += votes[0];
|
||||
voteDiagonal += votes[1];
|
||||
voteHorizontal += votes[2];
|
||||
for (const fingerSlope of fingerSlopes) {
|
||||
const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
|
||||
voteVertical += fingerVotes[0];
|
||||
voteDiagonal += fingerVotes[1];
|
||||
voteHorizontal += fingerVotes[2];
|
||||
}
|
||||
let estimatedDirection;
|
||||
if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
} else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
} else {
|
||||
estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
function estimate(landmarks) {
|
||||
const slopesXY = [];
|
||||
const slopesYZ = [];
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY = [];
|
||||
const slopeAtYZ = [];
|
||||
for (const point2 of points) {
|
||||
const point1 = landmarks[point2[0]];
|
||||
const point22 = landmarks[point2[1]];
|
||||
const slopes = getSlopes(point1, point22);
|
||||
const slopeXY = slopes[0];
|
||||
const slopeYZ = slopes[1];
|
||||
slopeAtXY.push(slopeXY);
|
||||
slopeAtYZ.push(slopeYZ);
|
||||
}
|
||||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
const fingerCurls = [];
|
||||
const fingerDirections = [];
|
||||
for (const finger of Finger.all) {
|
||||
const pointIndexAt = finger === Finger.thumb ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
|
||||
const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
|
||||
const endPoint = landmarks[fingerPointsAt[3][1]];
|
||||
const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
|
||||
const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
|
||||
fingerCurls[finger] = fingerCurled;
|
||||
fingerDirections[finger] = fingerPosition;
|
||||
}
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
}
|
||||
|
||||
// src/fingerpose/gesture.ts
|
||||
var Gesture = class {
|
||||
constructor(name) {
|
||||
this.name = name;
|
||||
this.curls = {};
|
||||
this.directions = {};
|
||||
this.weights = [1, 1, 1, 1, 1];
|
||||
this.weightsRelative = [1, 1, 1, 1, 1];
|
||||
}
|
||||
addCurl(finger, curl, confidence) {
|
||||
if (typeof this.curls[finger] === "undefined")
|
||||
this.curls[finger] = [];
|
||||
this.curls[finger].push([curl, confidence]);
|
||||
}
|
||||
addDirection(finger, position, confidence) {
|
||||
if (!this.directions[finger])
|
||||
this.directions[finger] = [];
|
||||
this.directions[finger].push([position, confidence]);
|
||||
}
|
||||
setWeight(finger, weight) {
|
||||
this.weights[finger] = weight;
|
||||
const total = this.weights.reduce((a, b) => a + b, 0);
|
||||
this.weightsRelative = this.weights.map((el) => el * 5 / total);
|
||||
}
|
||||
matchAgainst(detectedCurls, detectedDirections) {
|
||||
let confidence = 0;
|
||||
for (const fingerIdx in detectedCurls) {
|
||||
const detectedCurl = detectedCurls[fingerIdx];
|
||||
const expectedCurls = this.curls[fingerIdx];
|
||||
if (typeof expectedCurls === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedCurl, score3] of expectedCurls) {
|
||||
if (detectedCurl === expectedCurl) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const fingerIdx in detectedDirections) {
|
||||
const detectedDirection = detectedDirections[fingerIdx];
|
||||
const expectedDirections = this.directions[fingerIdx];
|
||||
if (typeof expectedDirections === "undefined") {
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
for (const [expectedDirection, score3] of expectedDirections) {
|
||||
if (detectedDirection === expectedDirection) {
|
||||
confidence += score3 * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return confidence / 10;
|
||||
}
|
||||
};
|
||||
|
||||
// src/fingerpose/gestures.ts
|
||||
var ThumbsUp = new Gesture("thumbs up");
|
||||
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
|
||||
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
|
||||
ThumbsUp.addCurl(finger, FingerCurl.full, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1);
|
||||
}
|
||||
var Victory = new Gesture("victory");
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.index, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
|
||||
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addCurl(Finger.middle, FingerCurl.none, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
|
||||
Victory.addCurl(Finger.ring, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.addCurl(Finger.pinky, FingerCurl.full, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.setWeight(Finger.index, 2);
|
||||
Victory.setWeight(Finger.middle, 2);
|
||||
var gestures_default = [ThumbsUp, Victory];
|
||||
|
||||
// src/fingerpose/fingerpose.ts
|
||||
var minConfidence = 0.7;
|
||||
function analyze(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const landmarks = {};
|
||||
for (const fingerIdx of Finger.all) {
|
||||
landmarks[Finger.getName(fingerIdx)] = {
|
||||
curl: FingerCurl.getName(estimatorRes.curls[fingerIdx]),
|
||||
direction: FingerDirection.getName(estimatorRes.directions[fingerIdx])
|
||||
};
|
||||
}
|
||||
return landmarks;
|
||||
}
|
||||
function match2(keypoints3) {
|
||||
const estimatorRes = estimate(keypoints3);
|
||||
const poses2 = [];
|
||||
for (const gesture3 of gestures_default) {
|
||||
const confidence = gesture3.matchAgainst(estimatorRes.curls, estimatorRes.directions);
|
||||
if (confidence >= minConfidence)
|
||||
poses2.push({ name: gesture3.name, confidence });
|
||||
}
|
||||
return poses2;
|
||||
}
|
||||
|
||||
// src/handpose/handpose.ts
|
||||
var meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
indexFinger: [5, 6, 7, 8],
|
||||
middleFinger: [9, 10, 11, 12],
|
||||
ringFinger: [13, 14, 15, 16],
|
||||
index: [5, 6, 7, 8],
|
||||
middle: [9, 10, 11, 12],
|
||||
ring: [13, 14, 15, 16],
|
||||
pinky: [17, 18, 19, 20],
|
||||
palmBase: [0]
|
||||
palm: [0]
|
||||
};
|
||||
var handDetectorModel;
|
||||
var handPoseModel;
|
||||
|
|
@ -8014,7 +8383,16 @@ async function predict5(input, config3) {
|
|||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0)
|
||||
];
|
||||
}
|
||||
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box: box6, boxRaw: boxRaw3, keypoints: keypoints3, annotations: annotations3 });
|
||||
const landmarks = analyze(keypoints3);
|
||||
hands.push({
|
||||
id: i,
|
||||
score: Math.round(100 * predictions[i].confidence) / 100,
|
||||
box: box6,
|
||||
boxRaw: boxRaw3,
|
||||
keypoints: keypoints3,
|
||||
annotations: annotations3,
|
||||
landmarks
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
@ -8706,9 +9084,9 @@ var tf18 = __toModule(require_tfjs_esm());
|
|||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
const _collect = function(source, prefix, collection) {
|
||||
const r = new RegExp("\\b" + prefix + " \\w+ (\\w+)", "ig");
|
||||
source.replace(r, (match2, name) => {
|
||||
source.replace(r, (match3, name) => {
|
||||
collection[name] = 0;
|
||||
return match2;
|
||||
return match3;
|
||||
});
|
||||
};
|
||||
const _compile = function(source, type) {
|
||||
|
|
@ -10013,6 +10391,9 @@ var hand = (res) => {
|
|||
const highest = fingers.reduce((best, a) => best.position[1] < a.position[1] ? best : a);
|
||||
gestures.push({ hand: i, gesture: `${highest.name} up` });
|
||||
}
|
||||
const poses2 = match2(res[i]["keypoints"]);
|
||||
for (const pose of poses2)
|
||||
gestures.push({ hand: i, gesture: pose.name });
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
@ -10027,10 +10408,10 @@ __export(draw_exports, {
|
|||
gesture: () => gesture,
|
||||
hand: () => hand2,
|
||||
object: () => object,
|
||||
options: () => options,
|
||||
options: () => options2,
|
||||
person: () => person
|
||||
});
|
||||
var options = {
|
||||
var options2 = {
|
||||
color: "rgba(173, 216, 230, 0.6)",
|
||||
labelColor: "rgba(173, 216, 230, 1)",
|
||||
shadowColor: "black",
|
||||
|
|
@ -10115,7 +10496,7 @@ function curves(ctx, points = [], localOptions) {
|
|||
}
|
||||
}
|
||||
async function gesture(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10145,7 +10526,7 @@ async function gesture(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function face2(inCanvas2, result, drawOptions) {
|
||||
var _a, _b, _c, _d;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10255,7 +10636,7 @@ async function face2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
async function body2(inCanvas2, result, drawOptions) {
|
||||
var _a;
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10387,7 +10768,7 @@ async function body2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function hand2(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10426,12 +10807,12 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations["indexFinger"], "index");
|
||||
addHandLabel(h.annotations["middleFinger"], "middle");
|
||||
addHandLabel(h.annotations["ringFinger"], "ring");
|
||||
addHandLabel(h.annotations["index"], "index");
|
||||
addHandLabel(h.annotations["middle"], "middle");
|
||||
addHandLabel(h.annotations["ring"], "ring");
|
||||
addHandLabel(h.annotations["pinky"], "pinky");
|
||||
addHandLabel(h.annotations["thumb"], "thumb");
|
||||
addHandLabel(h.annotations["palmBase"], "palm");
|
||||
addHandLabel(h.annotations["palm"], "palm");
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -10446,16 +10827,16 @@ async function hand2(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations["indexFinger"]);
|
||||
addHandLine(h.annotations["middleFinger"]);
|
||||
addHandLine(h.annotations["ringFinger"]);
|
||||
addHandLine(h.annotations["index"]);
|
||||
addHandLine(h.annotations["middle"]);
|
||||
addHandLine(h.annotations["ring"]);
|
||||
addHandLine(h.annotations["pinky"]);
|
||||
addHandLine(h.annotations["thumb"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function object(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10484,7 +10865,7 @@ async function object(inCanvas2, result, drawOptions) {
|
|||
}
|
||||
}
|
||||
async function person(inCanvas2, result, drawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
@ -10522,7 +10903,7 @@ async function canvas(inCanvas2, outCanvas2) {
|
|||
}
|
||||
async function all(inCanvas2, result, drawOptions) {
|
||||
const timestamp = now();
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
const localOptions = mergeDeep(options2, drawOptions);
|
||||
if (!result || !inCanvas2)
|
||||
return null;
|
||||
if (!(inCanvas2 instanceof HTMLCanvasElement))
|
||||
|
|
|
|||
|
|
@ -1,22 +1,22 @@
|
|||
2021-08-20 09:03:11 [36mINFO: [39m @vladmandic/human version 2.1.4
|
||||
2021-08-20 09:03:11 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-08-20 09:03:11 [36mINFO: [39m Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.21","typescript":"4.3.5","typedoc":"0.21.6","eslint":"7.32.0"}
|
||||
2021-08-20 09:03:11 [36mINFO: [39m Clean: ["dist/*","types/*","typedoc/*"]
|
||||
2021-08-20 09:03:11 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: node type: node: {"imports":42,"importBytes":438886,"outputBytes":381429,"outputFiles":"dist/human.node.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: nodeGPU type: node: {"imports":42,"importBytes":438894,"outputBytes":381433,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: nodeWASM type: node: {"imports":42,"importBytes":438961,"outputBytes":381505,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: browserNoBundle type: esm: {"imports":42,"importBytes":438825,"outputBytes":249942,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-08-20 09:03:12 [35mSTATE:[39m target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 09:03:13 [35mSTATE:[39m target: browserBundle type: iife: {"imports":42,"importBytes":2772284,"outputBytes":1380316,"outputFiles":"dist/human.js"}
|
||||
2021-08-20 09:03:13 [35mSTATE:[39m target: browserBundle type: esm: {"imports":42,"importBytes":2772284,"outputBytes":1380308,"outputFiles":"dist/human.esm.js"}
|
||||
2021-08-20 09:03:13 [36mINFO: [39m Running Linter: ["server/","src/","tfjs/","test/","demo/"]
|
||||
2021-08-20 09:03:37 [36mINFO: [39m Linter complete: files: 77 errors: 0 warnings: 0
|
||||
2021-08-20 09:03:37 [36mINFO: [39m Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-08-20 09:03:37 [36mINFO: [39m Generate Typings: ["src/human.ts"] outDir: ["types"]
|
||||
2021-08-20 09:03:51 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
|
||||
2021-08-20 09:04:05 [36mINFO: [39m Documentation generated at /home/vlado/dev/human/typedoc 1
|
||||
2021-08-20 20:41:20 [36mINFO: [39m @vladmandic/human version 2.1.4
|
||||
2021-08-20 20:41:20 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-08-20 20:41:20 [36mINFO: [39m Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.21","typescript":"4.3.5","typedoc":"0.21.6","eslint":"7.32.0"}
|
||||
2021-08-20 20:41:20 [36mINFO: [39m Clean: ["dist/*","types/*","typedoc/*"]
|
||||
2021-08-20 20:41:20 [36mINFO: [39m Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: node type: node: {"imports":47,"importBytes":456494,"outputBytes":396866,"outputFiles":"dist/human.node.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: nodeGPU type: node: {"imports":47,"importBytes":456502,"outputBytes":396870,"outputFiles":"dist/human.node-gpu.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: nodeWASM type: node: {"imports":47,"importBytes":456569,"outputBytes":396942,"outputFiles":"dist/human.node-wasm.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 20:41:20 [35mSTATE:[39m target: browserNoBundle type: esm: {"imports":47,"importBytes":456433,"outputBytes":255552,"outputFiles":"dist/human.esm-nobundle.js"}
|
||||
2021-08-20 20:41:21 [35mSTATE:[39m target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
|
||||
2021-08-20 20:41:21 [35mSTATE:[39m target: browserBundle type: iife: {"imports":47,"importBytes":2789892,"outputBytes":1386025,"outputFiles":"dist/human.js"}
|
||||
2021-08-20 20:41:21 [35mSTATE:[39m target: browserBundle type: esm: {"imports":47,"importBytes":2789892,"outputBytes":1386017,"outputFiles":"dist/human.esm.js"}
|
||||
2021-08-20 20:41:21 [36mINFO: [39m Running Linter: ["server/","src/","tfjs/","test/","demo/"]
|
||||
2021-08-20 20:41:44 [36mINFO: [39m Linter complete: files: 82 errors: 0 warnings: 0
|
||||
2021-08-20 20:41:44 [36mINFO: [39m Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
|
||||
2021-08-20 20:41:44 [36mINFO: [39m Generate Typings: ["src/human.ts"] outDir: ["types"]
|
||||
2021-08-20 20:41:58 [36mINFO: [39m Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
|
||||
2021-08-20 20:42:12 [36mINFO: [39m Documentation generated at /home/vlado/dev/human/typedoc 1
|
||||
|
|
|
|||
|
|
@ -414,12 +414,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations['indexFinger'], 'index');
|
||||
addHandLabel(h.annotations['middleFinger'], 'middle');
|
||||
addHandLabel(h.annotations['ringFinger'], 'ring');
|
||||
addHandLabel(h.annotations['index'], 'index');
|
||||
addHandLabel(h.annotations['middle'], 'middle');
|
||||
addHandLabel(h.annotations['ring'], 'ring');
|
||||
addHandLabel(h.annotations['pinky'], 'pinky');
|
||||
addHandLabel(h.annotations['thumb'], 'thumb');
|
||||
addHandLabel(h.annotations['palmBase'], 'palm');
|
||||
addHandLabel(h.annotations['palm'], 'palm');
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -433,12 +433,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations['indexFinger']);
|
||||
addHandLine(h.annotations['middleFinger']);
|
||||
addHandLine(h.annotations['ringFinger']);
|
||||
addHandLine(h.annotations['index']);
|
||||
addHandLine(h.annotations['middle']);
|
||||
addHandLine(h.annotations['ring']);
|
||||
addHandLine(h.annotations['pinky']);
|
||||
addHandLine(h.annotations['thumb']);
|
||||
// addPart(h.annotations.palmBase);
|
||||
// addPart(h.annotations.palm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
const Finger = {
|
||||
thumb: 0,
|
||||
index: 1,
|
||||
middle: 2,
|
||||
ring: 3,
|
||||
pinky: 4,
|
||||
all: [0, 1, 2, 3, 4], // just for convenience
|
||||
nameMapping: { 0: 'thumb', 1: 'index', 2: 'middle', 3: 'ring', 4: 'pinky' },
|
||||
// Describes mapping of joints based on the 21 points returned by handpose.
|
||||
// [0] Palm
|
||||
// [1-4] Thumb
|
||||
// [5-8] Index
|
||||
// [9-12] Middle
|
||||
// [13-16] Ring
|
||||
// [17-20] Pinky
|
||||
pointsMapping: {
|
||||
0: [[0, 1], [1, 2], [2, 3], [3, 4]],
|
||||
1: [[0, 5], [5, 6], [6, 7], [7, 8]],
|
||||
2: [[0, 9], [9, 10], [10, 11], [11, 12]],
|
||||
3: [[0, 13], [13, 14], [14, 15], [15, 16]],
|
||||
4: [[0, 17], [17, 18], [18, 19], [19, 20]],
|
||||
},
|
||||
getName: (value) => Finger.nameMapping[value],
|
||||
getPoints: (value) => Finger.pointsMapping[value],
|
||||
};
|
||||
|
||||
const FingerCurl = {
|
||||
none: 0,
|
||||
half: 1,
|
||||
full: 2,
|
||||
nameMapping: { 0: 'none', 1: 'half', 2: 'full' },
|
||||
getName: (value) => FingerCurl.nameMapping[value],
|
||||
};
|
||||
|
||||
const FingerDirection = {
|
||||
verticalUp: 0,
|
||||
verticalDown: 1,
|
||||
horizontalLeft: 2,
|
||||
horizontalRight: 3,
|
||||
diagonalUpRight: 4,
|
||||
diagonalUpLeft: 5,
|
||||
diagonalDownRight: 6,
|
||||
diagonalDownLeft: 7,
|
||||
nameMapping: { 0: 'verticalUp', 1: 'verticalDown', 2: 'horizontalLeft', 3: 'horizontalRight', 4: 'diagonalUpRight', 5: 'diagonalUpLeft', 6: 'diagonalDownRight', 7: 'diagonalDownLeft' },
|
||||
getName: (value) => FingerDirection.nameMapping[value],
|
||||
};
|
||||
|
||||
export { Finger, FingerCurl, FingerDirection };
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
import { Finger, FingerCurl, FingerDirection } from './description';
|
||||
|
||||
const options = {
|
||||
// curl estimation
|
||||
HALF_CURL_START_LIMIT: 60.0,
|
||||
NO_CURL_START_LIMIT: 130.0,
|
||||
// direction estimation
|
||||
DISTANCE_VOTE_POWER: 1.1,
|
||||
SINGLE_ANGLE_VOTE_POWER: 0.9,
|
||||
TOTAL_ANGLE_VOTE_POWER: 1.6,
|
||||
};
|
||||
|
||||
function calculateSlope(point1x, point1y, point2x, point2y) {
|
||||
const value = (point1y - point2y) / (point1x - point2x);
|
||||
let slope = Math.atan(value) * 180 / Math.PI;
|
||||
if (slope <= 0) slope = -slope;
|
||||
else if (slope > 0) slope = 180 - slope;
|
||||
return slope;
|
||||
}
|
||||
|
||||
// point1, point2 are 2d or 3d point arrays (xy[z])
|
||||
// returns either a single scalar (2d) or array of two slopes (3d)
|
||||
function getSlopes(point1, point2) {
|
||||
const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
|
||||
if (point1.length === 2) return slopeXY;
|
||||
const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
|
||||
return [slopeXY, slopeYZ];
|
||||
}
|
||||
|
||||
function angleOrientationAt(angle, weightageAt = 1.0) {
|
||||
let isVertical = 0;
|
||||
let isDiagonal = 0;
|
||||
let isHorizontal = 0;
|
||||
if (angle >= 75.0 && angle <= 105.0) isVertical = 1 * weightageAt;
|
||||
else if (angle >= 25.0 && angle <= 155.0) isDiagonal = 1 * weightageAt;
|
||||
else isHorizontal = 1 * weightageAt;
|
||||
return [isVertical, isDiagonal, isHorizontal];
|
||||
}
|
||||
|
||||
function estimateFingerCurl(startPoint, midPoint, endPoint) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const start_mid_z_dist = startPoint[2] - midPoint[2];
|
||||
const start_end_z_dist = startPoint[2] - endPoint[2];
|
||||
const mid_end_z_dist = midPoint[2] - endPoint[2];
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
|
||||
let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
|
||||
if (cos_in > 1.0) cos_in = 1.0;
|
||||
else if (cos_in < -1.0) cos_in = -1.0;
|
||||
let angleOfCurve = Math.acos(cos_in);
|
||||
angleOfCurve = (57.2958 * angleOfCurve) % 180;
|
||||
let fingerCurl;
|
||||
if (angleOfCurve > options.NO_CURL_START_LIMIT) fingerCurl = FingerCurl.none;
|
||||
else if (angleOfCurve > options.HALF_CURL_START_LIMIT) fingerCurl = FingerCurl.half;
|
||||
else fingerCurl = FingerCurl.full;
|
||||
return fingerCurl;
|
||||
}
|
||||
|
||||
function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_x === Math.abs(start_end_x_dist)) {
|
||||
if (start_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else if (max_dist_x === Math.abs(start_mid_x_dist)) {
|
||||
if (start_mid_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else {
|
||||
if (mid_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_y === Math.abs(start_end_y_dist)) {
|
||||
if (start_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
} else if (max_dist_y === Math.abs(start_mid_y_dist)) {
|
||||
if (start_mid_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
} else {
|
||||
if (mid_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
if (reqd_vertical_direction === FingerDirection.verticalUp) {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalUpLeft;
|
||||
else estimatedDirection = FingerDirection.diagonalUpRight;
|
||||
} else {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalDownLeft;
|
||||
else estimatedDirection = FingerDirection.diagonalDownRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
|
||||
const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
|
||||
let voteVertical = 0.0;
|
||||
let voteDiagonal = 0.0;
|
||||
let voteHorizontal = 0.0;
|
||||
const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 0.00001);
|
||||
if (start_end_x_y_dist_ratio > 1.5) voteVertical += options.DISTANCE_VOTE_POWER;
|
||||
else if (start_end_x_y_dist_ratio > 0.66) voteDiagonal += options.DISTANCE_VOTE_POWER;
|
||||
else voteHorizontal += options.DISTANCE_VOTE_POWER;
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
|
||||
const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
|
||||
let calc_start_point_x = startPoint[0];
|
||||
let calc_start_point_y = startPoint[1];
|
||||
let calc_end_point_x = endPoint[0];
|
||||
let calc_end_point_y = endPoint[1];
|
||||
if (max_dist === start_mid_dist) {
|
||||
calc_end_point_x = endPoint[0];
|
||||
calc_end_point_y = endPoint[1];
|
||||
} else if (max_dist === mid_end_dist) {
|
||||
calc_start_point_x = midPoint[0];
|
||||
calc_start_point_y = midPoint[1];
|
||||
}
|
||||
const calcStartPoint = [calc_start_point_x, calc_start_point_y];
|
||||
const calcEndPoint = [calc_end_point_x, calc_end_point_y];
|
||||
const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
|
||||
const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
|
||||
voteVertical += votes[0];
|
||||
voteDiagonal += votes[1];
|
||||
voteHorizontal += votes[2];
|
||||
for (const fingerSlope of fingerSlopes) {
|
||||
const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
|
||||
voteVertical += fingerVotes[0];
|
||||
voteDiagonal += fingerVotes[1];
|
||||
voteHorizontal += fingerVotes[2];
|
||||
}
|
||||
// in case of tie, highest preference goes to Vertical,
|
||||
// followed by horizontal and then diagonal
|
||||
let estimatedDirection;
|
||||
if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
} else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
} else {
|
||||
estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
export function estimate(landmarks) {
|
||||
// step 1: calculate slopes
|
||||
const slopesXY: Array<number[]> = [];
|
||||
const slopesYZ: Array<number[]> = [];
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY: Array<number> = [];
|
||||
const slopeAtYZ: Array<number> = [];
|
||||
for (const point of points) {
|
||||
const point1 = landmarks[point[0]];
|
||||
const point2 = landmarks[point[1]];
|
||||
// calculate single slope
|
||||
const slopes = getSlopes(point1, point2);
|
||||
const slopeXY = slopes[0];
|
||||
const slopeYZ = slopes[1];
|
||||
slopeAtXY.push(slopeXY);
|
||||
slopeAtYZ.push(slopeYZ);
|
||||
}
|
||||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
|
||||
// step 2: calculate orientations
|
||||
const fingerCurls: Array<number> = [];
|
||||
const fingerDirections: Array<number> = [];
|
||||
for (const finger of Finger.all) {
|
||||
// start finger predictions from palm - except for thumb
|
||||
const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
|
||||
const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
|
||||
const endPoint = landmarks[fingerPointsAt[3][1]];
|
||||
// check if finger is curled
|
||||
const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
|
||||
const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
|
||||
fingerCurls[finger] = fingerCurled;
|
||||
fingerDirections[finger] = fingerPosition;
|
||||
}
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
// based on <https://github.com/andypotato/fingerpose>
|
||||
|
||||
import * as estimator from './estimator';
|
||||
import { Finger, FingerCurl, FingerDirection } from './description';
|
||||
import Gestures from './gestures';
|
||||
|
||||
const minConfidence = 0.7;
|
||||
|
||||
export function analyze(keypoints) { // get estimations of curl / direction for each finger
|
||||
const estimatorRes = estimator.estimate(keypoints);
|
||||
const landmarks = {};
|
||||
for (const fingerIdx of Finger.all) {
|
||||
landmarks[Finger.getName(fingerIdx)] = {
|
||||
curl: FingerCurl.getName(estimatorRes.curls[fingerIdx]),
|
||||
direction: FingerDirection.getName(estimatorRes.directions[fingerIdx]),
|
||||
};
|
||||
}
|
||||
// console.log('finger landmarks', landmarks);
|
||||
return landmarks;
|
||||
}
|
||||
|
||||
export function match(keypoints) { // compare gesture description to each known gesture
|
||||
const estimatorRes = estimator.estimate(keypoints);
|
||||
const poses: Array<{ name: string, confidence: number }> = [];
|
||||
for (const gesture of Gestures) {
|
||||
const confidence = gesture.matchAgainst(estimatorRes.curls, estimatorRes.directions);
|
||||
if (confidence >= minConfidence) poses.push({ name: gesture.name, confidence });
|
||||
}
|
||||
// console.log('finger poses', poses);
|
||||
return poses;
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
export default class Gesture {
|
||||
name;
|
||||
curls;
|
||||
directions;
|
||||
weights;
|
||||
weightsRelative;
|
||||
|
||||
constructor(name) {
|
||||
// name (should be unique)
|
||||
this.name = name;
|
||||
this.curls = {};
|
||||
this.directions = {};
|
||||
this.weights = [1.0, 1.0, 1.0, 1.0, 1.0];
|
||||
this.weightsRelative = [1.0, 1.0, 1.0, 1.0, 1.0];
|
||||
}
|
||||
|
||||
addCurl(finger, curl, confidence) {
|
||||
if (typeof this.curls[finger] === 'undefined') this.curls[finger] = [];
|
||||
this.curls[finger].push([curl, confidence]);
|
||||
}
|
||||
|
||||
addDirection(finger, position, confidence) {
|
||||
if (!this.directions[finger]) this.directions[finger] = [];
|
||||
this.directions[finger].push([position, confidence]);
|
||||
}
|
||||
|
||||
setWeight(finger, weight) {
|
||||
this.weights[finger] = weight;
|
||||
// recalculate relative weights
|
||||
const total = this.weights.reduce((a, b) => a + b, 0);
|
||||
this.weightsRelative = this.weights.map((el) => el * 5 / total);
|
||||
}
|
||||
|
||||
matchAgainst(detectedCurls, detectedDirections) {
|
||||
let confidence = 0.0;
|
||||
// look at the detected curl of each finger and compare with
|
||||
// the expected curl of this finger inside current gesture
|
||||
for (const fingerIdx in detectedCurls) {
|
||||
const detectedCurl = detectedCurls[fingerIdx];
|
||||
const expectedCurls = this.curls[fingerIdx];
|
||||
if (typeof expectedCurls === 'undefined') {
|
||||
// no curl description available for this finger
|
||||
// add default confidence of "1"
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
// compare to each possible curl of this specific finger
|
||||
for (const [expectedCurl, score] of expectedCurls) {
|
||||
if (detectedCurl === expectedCurl) {
|
||||
confidence += score * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// same for detected direction of each finger
|
||||
for (const fingerIdx in detectedDirections) {
|
||||
const detectedDirection = detectedDirections[fingerIdx];
|
||||
const expectedDirections = this.directions[fingerIdx];
|
||||
if (typeof expectedDirections === 'undefined') {
|
||||
// no direction description available for this finger
|
||||
// add default confidence of "1"
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
// compare to each possible direction of this specific finger
|
||||
for (const [expectedDirection, score] of expectedDirections) {
|
||||
if (detectedDirection === expectedDirection) {
|
||||
confidence += score * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return confidence / 10;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import { Finger, FingerCurl, FingerDirection } from './description';
|
||||
import Gesture from './gesture';
|
||||
|
||||
// describe thumbs up gesture 👍
|
||||
const ThumbsUp = new Gesture('thumbs up');
|
||||
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1.0);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
|
||||
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
|
||||
ThumbsUp.addCurl(finger, FingerCurl.full, 1.0);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1.0);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1.0);
|
||||
}
|
||||
|
||||
// describe Victory gesture ✌️
|
||||
const Victory = new Gesture('victory');
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addCurl(Finger.index, FingerCurl.none, 1.0);
|
||||
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
|
||||
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addCurl(Finger.middle, FingerCurl.none, 1.0);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1.0);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
|
||||
Victory.addCurl(Finger.ring, FingerCurl.full, 1.0);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.addCurl(Finger.pinky, FingerCurl.full, 1.0);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.setWeight(Finger.index, 2);
|
||||
Victory.setWeight(Finger.middle, 2);
|
||||
|
||||
export default [ThumbsUp, Victory];
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
*/
|
||||
|
||||
import { Gesture } from '../result';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
|
||||
/**
|
||||
* @typedef FaceGesture
|
||||
|
|
@ -33,8 +34,10 @@ export type BodyGesture =
|
|||
* @typedef BodyGesture
|
||||
*/
|
||||
export type HandGesture =
|
||||
`${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} forward`
|
||||
| `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} up`;
|
||||
`${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} forward`
|
||||
| `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} up`
|
||||
| 'victory'
|
||||
| 'thumbs up';
|
||||
|
||||
export const body = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
|
|
@ -129,6 +132,8 @@ export const hand = (res): Gesture[] => {
|
|||
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
|
||||
gestures.push({ hand: i, gesture: `${highest.name} up` as HandGesture });
|
||||
}
|
||||
const poses = fingerPose.match(res[i]['keypoints']);
|
||||
for (const pose of poses) gestures.push({ hand: i, gesture: pose.name as HandGesture });
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,17 +6,18 @@ import { log, join } from '../helpers';
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as handdetector from './handdetector';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
import { Hand } from '../result';
|
||||
import { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
const meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
indexFinger: [5, 6, 7, 8],
|
||||
middleFinger: [9, 10, 11, 12],
|
||||
ringFinger: [13, 14, 15, 16],
|
||||
index: [5, 6, 7, 8],
|
||||
middle: [9, 10, 11, 12],
|
||||
ring: [13, 14, 15, 16],
|
||||
pinky: [17, 18, 19, 20],
|
||||
palmBase: [0],
|
||||
palm: [0],
|
||||
};
|
||||
|
||||
let handDetectorModel: GraphModel | null;
|
||||
|
|
@ -64,7 +65,16 @@ export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
|
|||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0),
|
||||
];
|
||||
}
|
||||
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations });
|
||||
const landmarks = fingerPose.analyze(keypoints);
|
||||
hands.push({
|
||||
id: i,
|
||||
score: Math.round(100 * predictions[i].confidence) / 100,
|
||||
box,
|
||||
boxRaw,
|
||||
keypoints,
|
||||
annotations: annotations as Hand['annotations'],
|
||||
landmarks: landmarks as Hand['landmarks'],
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ export function calc(newResult: Result): Result {
|
|||
annotations[key] = newResult.hand[i].annotations[key]
|
||||
.map((val, j) => val.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor));
|
||||
}
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations }; // shallow clone plus updated values
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as Hand['annotations'] }; // shallow clone plus updated values
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -90,8 +90,9 @@ export interface Body {
|
|||
* - score: detection confidence score as value
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part
|
||||
* - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part with keypoints
|
||||
* - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
|
||||
*/
|
||||
export interface Hand {
|
||||
id: number,
|
||||
|
|
@ -99,7 +100,14 @@ export interface Hand {
|
|||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
keypoints: Array<[number, number, number]>,
|
||||
annotations: Record<string, Array<[number, number, number]>>,
|
||||
annotations: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
|
||||
Array<[number, number, number]>
|
||||
>,
|
||||
landmarks: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
|
||||
{ curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
|
||||
>,
|
||||
}
|
||||
|
||||
/** Object results
|
||||
|
|
|
|||
240
test/test.log
240
test/test.log
|
|
@ -1,120 +1,120 @@
|
|||
2021-08-20 09:04:08 [36mINFO: [39m @vladmandic/human version 2.1.4
|
||||
2021-08-20 09:04:08 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-08-20 09:04:08 [36mINFO: [39m tests: ["test-node.js","test-node-gpu.js","test-node-wasm.js"]
|
||||
2021-08-20 09:04:08 [36mINFO: [39m test-node.js start
|
||||
2021-08-20 09:04:08 [35mSTATE:[39m test-node.js passed: create human
|
||||
2021-08-20 09:04:08 [36mINFO: [39m test-node.js human version: 2.1.4
|
||||
2021-08-20 09:04:08 [36mINFO: [39m test-node.js platform: linux x64 agent: NodeJS v16.5.0
|
||||
2021-08-20 09:04:08 [36mINFO: [39m test-node.js tfjs version: 3.8.0
|
||||
2021-08-20 09:04:09 [35mSTATE:[39m test-node.js passed: set backend: tensorflow
|
||||
2021-08-20 09:04:09 [35mSTATE:[39m test-node.js passed: load models
|
||||
2021-08-20 09:04:09 [35mSTATE:[39m test-node.js result: defined models: 14 loaded models: 7
|
||||
2021-08-20 09:04:09 [35mSTATE:[39m test-node.js passed: warmup: none default
|
||||
2021-08-20 09:04:10 [35mSTATE:[39m test-node.js passed: warmup: face default
|
||||
2021-08-20 09:04:10 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.42,"keypoints":4}
|
||||
2021-08-20 09:04:10 [32mDATA: [39m test-node.js result: performance: load: 372 total: 1090
|
||||
2021-08-20 09:04:11 [35mSTATE:[39m test-node.js passed: warmup: body default
|
||||
2021-08-20 09:04:11 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:11 [32mDATA: [39m test-node.js result: performance: load: 372 total: 1054
|
||||
2021-08-20 09:04:11 [36mINFO: [39m test-node.js test body variants
|
||||
2021-08-20 09:04:12 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:13 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg posenet
|
||||
2021-08-20 09:04:13 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.96,"keypoints":16}
|
||||
2021-08-20 09:04:13 [32mDATA: [39m test-node.js result: performance: load: 372 total: 687
|
||||
2021-08-20 09:04:13 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:14 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg movenet
|
||||
2021-08-20 09:04:14 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:14 [32mDATA: [39m test-node.js result: performance: load: 372 total: 193
|
||||
2021-08-20 09:04:14 [35mSTATE:[39m test-node.js passed: detect: random default
|
||||
2021-08-20 09:04:14 [32mDATA: [39m test-node.js result: face: 0 body: 1 hand: 0 gesture: 0 object: 0 person: 0 {} {} {"score":0,"keypoints":0}
|
||||
2021-08-20 09:04:14 [32mDATA: [39m test-node.js result: performance: load: 372 total: 591
|
||||
2021-08-20 09:04:14 [36mINFO: [39m test-node.js test: first instance
|
||||
2021-08-20 09:04:15 [35mSTATE:[39m test-node.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 09:04:16 [35mSTATE:[39m test-node.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 09:04:16 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 09:04:16 [32mDATA: [39m test-node.js result: performance: load: 372 total: 975
|
||||
2021-08-20 09:04:16 [36mINFO: [39m test-node.js test: second instance
|
||||
2021-08-20 09:04:16 [35mSTATE:[39m test-node.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 09:04:17 [35mSTATE:[39m test-node.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 09:04:17 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 09:04:17 [32mDATA: [39m test-node.js result: performance: load: 3 total: 902
|
||||
2021-08-20 09:04:17 [36mINFO: [39m test-node.js test: concurrent
|
||||
2021-08-20 09:04:17 [35mSTATE:[39m test-node.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 09:04:17 [35mSTATE:[39m test-node.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 09:04:18 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:19 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:23 [35mSTATE:[39m test-node.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 4 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: performance: load: 372 total: 4140
|
||||
2021-08-20 09:04:23 [35mSTATE:[39m test-node.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 5 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: performance: load: 3 total: 4140
|
||||
2021-08-20 09:04:23 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: performance: load: 372 total: 4140
|
||||
2021-08-20 09:04:23 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:23 [32mDATA: [39m test-node.js result: performance: load: 3 total: 4140
|
||||
2021-08-20 09:04:23 [36mINFO: [39m test-node.js test complete: 14355 ms
|
||||
2021-08-20 09:04:23 [36mINFO: [39m test-node-gpu.js start
|
||||
2021-08-20 09:04:23 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 09:04:23.693064: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
|
||||
2021-08-20 09:04:23 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 09:04:23.741809: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory
|
||||
2021-08-20 09:04:23 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 09:04:23.741844: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (wyse): /proc/driver/nvidia/version does not exist
|
||||
2021-08-20 09:04:23 [35mSTATE:[39m test-node-gpu.js passed: create human
|
||||
2021-08-20 09:04:23 [36mINFO: [39m test-node-gpu.js human version: 2.1.4
|
||||
2021-08-20 09:04:23 [36mINFO: [39m test-node-gpu.js platform: linux x64 agent: NodeJS v16.5.0
|
||||
2021-08-20 09:04:23 [36mINFO: [39m test-node-gpu.js tfjs version: 3.8.0
|
||||
2021-08-20 09:04:24 [35mSTATE:[39m test-node-gpu.js passed: set backend: tensorflow
|
||||
2021-08-20 09:04:24 [35mSTATE:[39m test-node-gpu.js passed: load models
|
||||
2021-08-20 09:04:24 [35mSTATE:[39m test-node-gpu.js result: defined models: 14 loaded models: 7
|
||||
2021-08-20 09:04:24 [35mSTATE:[39m test-node-gpu.js passed: warmup: none default
|
||||
2021-08-20 09:04:25 [35mSTATE:[39m test-node-gpu.js passed: warmup: face default
|
||||
2021-08-20 09:04:25 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.42,"keypoints":4}
|
||||
2021-08-20 09:04:25 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 1107
|
||||
2021-08-20 09:04:26 [35mSTATE:[39m test-node-gpu.js passed: warmup: body default
|
||||
2021-08-20 09:04:26 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:26 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 1083
|
||||
2021-08-20 09:04:26 [36mINFO: [39m test-node-gpu.js test body variants
|
||||
2021-08-20 09:04:27 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:27 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg posenet
|
||||
2021-08-20 09:04:27 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.96,"keypoints":16}
|
||||
2021-08-20 09:04:27 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 728
|
||||
2021-08-20 09:04:28 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:28 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg movenet
|
||||
2021-08-20 09:04:28 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:28 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 191
|
||||
2021-08-20 09:04:29 [35mSTATE:[39m test-node-gpu.js passed: detect: random default
|
||||
2021-08-20 09:04:29 [32mDATA: [39m test-node-gpu.js result: face: 0 body: 1 hand: 0 gesture: 0 object: 0 person: 0 {} {} {"score":0,"keypoints":0}
|
||||
2021-08-20 09:04:29 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 608
|
||||
2021-08-20 09:04:29 [36mINFO: [39m test-node-gpu.js test: first instance
|
||||
2021-08-20 09:04:29 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 09:04:30 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 09:04:30 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 09:04:30 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 915
|
||||
2021-08-20 09:04:30 [36mINFO: [39m test-node-gpu.js test: second instance
|
||||
2021-08-20 09:04:31 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 09:04:32 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 09:04:32 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 09:04:32 [32mDATA: [39m test-node-gpu.js result: performance: load: 5 total: 982
|
||||
2021-08-20 09:04:32 [36mINFO: [39m test-node-gpu.js test: concurrent
|
||||
2021-08-20 09:04:32 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 09:04:32 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 09:04:32 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:33 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 09:04:38 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 4 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 4187
|
||||
2021-08-20 09:04:38 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 5 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: performance: load: 5 total: 4187
|
||||
2021-08-20 09:04:38 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: performance: load: 277 total: 4187
|
||||
2021-08-20 09:04:38 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 09:04:38 [32mDATA: [39m test-node-gpu.js result: performance: load: 5 total: 4187
|
||||
2021-08-20 09:04:38 [36mINFO: [39m test-node-gpu.js test complete: 14218 ms
|
||||
2021-08-20 09:04:38 [36mINFO: [39m test-node-wasm.js start
|
||||
2021-08-20 09:04:38 [31mERROR:[39m test-node-wasm.js failed: model server: request to http://localhost:10030/models/ failed, reason: connect ECONNREFUSED 127.0.0.1:10030
|
||||
2021-08-20 09:04:38 [31mERROR:[39m test-node-wasm.js aborting test
|
||||
2021-08-20 09:04:38 [36mINFO: [39m status: {"passed":46,"failed":1}
|
||||
2021-08-20 20:42:17 [36mINFO: [39m @vladmandic/human version 2.1.4
|
||||
2021-08-20 20:42:17 [36mINFO: [39m User: vlado Platform: linux Arch: x64 Node: v16.5.0
|
||||
2021-08-20 20:42:17 [36mINFO: [39m tests: ["test-node.js","test-node-gpu.js","test-node-wasm.js"]
|
||||
2021-08-20 20:42:17 [36mINFO: [39m test-node.js start
|
||||
2021-08-20 20:42:19 [35mSTATE:[39m test-node.js passed: create human
|
||||
2021-08-20 20:42:19 [36mINFO: [39m test-node.js human version: 2.1.4
|
||||
2021-08-20 20:42:19 [36mINFO: [39m test-node.js platform: linux x64 agent: NodeJS v16.5.0
|
||||
2021-08-20 20:42:19 [36mINFO: [39m test-node.js tfjs version: 3.8.0
|
||||
2021-08-20 20:42:19 [35mSTATE:[39m test-node.js passed: set backend: tensorflow
|
||||
2021-08-20 20:42:19 [35mSTATE:[39m test-node.js passed: load models
|
||||
2021-08-20 20:42:19 [35mSTATE:[39m test-node.js result: defined models: 14 loaded models: 7
|
||||
2021-08-20 20:42:19 [35mSTATE:[39m test-node.js passed: warmup: none default
|
||||
2021-08-20 20:42:21 [35mSTATE:[39m test-node.js passed: warmup: face default
|
||||
2021-08-20 20:42:21 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.42,"keypoints":4}
|
||||
2021-08-20 20:42:21 [32mDATA: [39m test-node.js result: performance: load: 343 total: 1358
|
||||
2021-08-20 20:42:22 [35mSTATE:[39m test-node.js passed: warmup: body default
|
||||
2021-08-20 20:42:22 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:22 [32mDATA: [39m test-node.js result: performance: load: 343 total: 1140
|
||||
2021-08-20 20:42:22 [36mINFO: [39m test-node.js test body variants
|
||||
2021-08-20 20:42:23 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:24 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg posenet
|
||||
2021-08-20 20:42:24 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.96,"keypoints":16}
|
||||
2021-08-20 20:42:24 [32mDATA: [39m test-node.js result: performance: load: 343 total: 695
|
||||
2021-08-20 20:42:24 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:25 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg movenet
|
||||
2021-08-20 20:42:25 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:25 [32mDATA: [39m test-node.js result: performance: load: 343 total: 257
|
||||
2021-08-20 20:42:26 [35mSTATE:[39m test-node.js passed: detect: random default
|
||||
2021-08-20 20:42:26 [32mDATA: [39m test-node.js result: face: 0 body: 1 hand: 0 gesture: 0 object: 0 person: 0 {} {} {"score":0,"keypoints":0}
|
||||
2021-08-20 20:42:26 [32mDATA: [39m test-node.js result: performance: load: 343 total: 704
|
||||
2021-08-20 20:42:26 [36mINFO: [39m test-node.js test: first instance
|
||||
2021-08-20 20:42:26 [35mSTATE:[39m test-node.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 20:42:27 [35mSTATE:[39m test-node.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 20:42:27 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 20:42:27 [32mDATA: [39m test-node.js result: performance: load: 343 total: 953
|
||||
2021-08-20 20:42:27 [36mINFO: [39m test-node.js test: second instance
|
||||
2021-08-20 20:42:27 [35mSTATE:[39m test-node.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 20:42:28 [35mSTATE:[39m test-node.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 20:42:28 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 20:42:28 [32mDATA: [39m test-node.js result: performance: load: 4 total: 940
|
||||
2021-08-20 20:42:28 [36mINFO: [39m test-node.js test: concurrent
|
||||
2021-08-20 20:42:28 [35mSTATE:[39m test-node.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 20:42:28 [35mSTATE:[39m test-node.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 20:42:29 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:30 [35mSTATE:[39m test-node.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:34 [35mSTATE:[39m test-node.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 4 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: performance: load: 343 total: 4377
|
||||
2021-08-20 20:42:34 [35mSTATE:[39m test-node.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 5 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: performance: load: 4 total: 4377
|
||||
2021-08-20 20:42:34 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: performance: load: 343 total: 4377
|
||||
2021-08-20 20:42:34 [35mSTATE:[39m test-node.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:34 [32mDATA: [39m test-node.js result: performance: load: 4 total: 4377
|
||||
2021-08-20 20:42:34 [36mINFO: [39m test-node.js test complete: 15280 ms
|
||||
2021-08-20 20:42:34 [36mINFO: [39m test-node-gpu.js start
|
||||
2021-08-20 20:42:35 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 20:42:35.983213: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
|
||||
2021-08-20 20:42:36 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 20:42:36.281966: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory
|
||||
2021-08-20 20:42:36 [33mWARN: [39m test-node-gpu.js stderr: 2021-08-20 20:42:36.282070: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (wyse): /proc/driver/nvidia/version does not exist
|
||||
2021-08-20 20:42:36 [35mSTATE:[39m test-node-gpu.js passed: create human
|
||||
2021-08-20 20:42:36 [36mINFO: [39m test-node-gpu.js human version: 2.1.4
|
||||
2021-08-20 20:42:36 [36mINFO: [39m test-node-gpu.js platform: linux x64 agent: NodeJS v16.5.0
|
||||
2021-08-20 20:42:36 [36mINFO: [39m test-node-gpu.js tfjs version: 3.8.0
|
||||
2021-08-20 20:42:36 [35mSTATE:[39m test-node-gpu.js passed: set backend: tensorflow
|
||||
2021-08-20 20:42:36 [35mSTATE:[39m test-node-gpu.js passed: load models
|
||||
2021-08-20 20:42:36 [35mSTATE:[39m test-node-gpu.js result: defined models: 14 loaded models: 7
|
||||
2021-08-20 20:42:36 [35mSTATE:[39m test-node-gpu.js passed: warmup: none default
|
||||
2021-08-20 20:42:38 [35mSTATE:[39m test-node-gpu.js passed: warmup: face default
|
||||
2021-08-20 20:42:38 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.42,"keypoints":4}
|
||||
2021-08-20 20:42:38 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 1504
|
||||
2021-08-20 20:42:39 [35mSTATE:[39m test-node-gpu.js passed: warmup: body default
|
||||
2021-08-20 20:42:39 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:39 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 1151
|
||||
2021-08-20 20:42:39 [36mINFO: [39m test-node-gpu.js test body variants
|
||||
2021-08-20 20:42:40 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:40 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg posenet
|
||||
2021-08-20 20:42:40 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.96,"keypoints":16}
|
||||
2021-08-20 20:42:40 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 735
|
||||
2021-08-20 20:42:41 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:42 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg movenet
|
||||
2021-08-20 20:42:42 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:42 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 222
|
||||
2021-08-20 20:42:42 [35mSTATE:[39m test-node-gpu.js passed: detect: random default
|
||||
2021-08-20 20:42:42 [32mDATA: [39m test-node-gpu.js result: face: 0 body: 1 hand: 0 gesture: 0 object: 0 person: 0 {} {} {"score":0,"keypoints":0}
|
||||
2021-08-20 20:42:42 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 752
|
||||
2021-08-20 20:42:42 [36mINFO: [39m test-node-gpu.js test: first instance
|
||||
2021-08-20 20:42:43 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 20:42:44 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 20:42:44 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 20:42:44 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 945
|
||||
2021-08-20 20:42:44 [36mINFO: [39m test-node-gpu.js test: second instance
|
||||
2021-08-20 20:42:44 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-upper.jpg [1,720,688,3]
|
||||
2021-08-20 20:42:45 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-upper.jpg default
|
||||
2021-08-20 20:42:45 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":29.5,"gender":"female"} {"score":0.71,"class":"person"} {"score":0.69,"keypoints":10}
|
||||
2021-08-20 20:42:45 [32mDATA: [39m test-node-gpu.js result: performance: load: 3 total: 914
|
||||
2021-08-20 20:42:45 [36mINFO: [39m test-node-gpu.js test: concurrent
|
||||
2021-08-20 20:42:45 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 20:42:45 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-face.jpg [1,256,256,3]
|
||||
2021-08-20 20:42:46 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:47 [35mSTATE:[39m test-node-gpu.js passed: load image: samples/ai-body.jpg [1,1200,1200,3]
|
||||
2021-08-20 20:42:51 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 4 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 4370
|
||||
2021-08-20 20:42:51 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-face.jpg default
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 5 object: 1 person: 1 {"age":23.6,"gender":"female"} {"score":0.82,"class":"person"} {"score":0.47,"keypoints":17}
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: performance: load: 3 total: 4370
|
||||
2021-08-20 20:42:51 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: performance: load: 289 total: 4370
|
||||
2021-08-20 20:42:51 [35mSTATE:[39m test-node-gpu.js passed: detect: samples/ai-body.jpg default
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: face: 1 body: 1 hand: 0 gesture: 3 object: 1 person: 1 {"age":28.5,"gender":"female"} {"score":0.72,"class":"person"} {"score":0.92,"keypoints":17}
|
||||
2021-08-20 20:42:51 [32mDATA: [39m test-node-gpu.js result: performance: load: 3 total: 4370
|
||||
2021-08-20 20:42:51 [36mINFO: [39m test-node-gpu.js test complete: 15164 ms
|
||||
2021-08-20 20:42:51 [36mINFO: [39m test-node-wasm.js start
|
||||
2021-08-20 20:42:51 [31mERROR:[39m test-node-wasm.js failed: model server: request to http://localhost:10030/models/ failed, reason: connect ECONNREFUSED 127.0.0.1:10030
|
||||
2021-08-20 20:42:51 [31mERROR:[39m test-node-wasm.js aborting test
|
||||
2021-08-20 20:42:51 [36mINFO: [39m status: {"passed":46,"failed":1}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -74,8 +74,9 @@
|
|||
<li>score: detection confidence score as value</li>
|
||||
<li>box: bounding box: x, y, width, height normalized to input image resolution</li>
|
||||
<li>boxRaw: bounding box: x, y, width, height normalized to 0..1</li>
|
||||
<li>landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution</li>
|
||||
<li>annotations: annotated landmarks for each hand part</li>
|
||||
<li>keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution</li>
|
||||
<li>annotations: annotated landmarks for each hand part with keypoints</li>
|
||||
<li>landmarks: annotated landmarks for eachb hand part with logical curl and direction strings</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
|
@ -99,6 +100,7 @@
|
|||
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="Hand.html#boxRaw" class="tsd-kind-icon">box<wbr>Raw</a></li>
|
||||
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="Hand.html#id" class="tsd-kind-icon">id</a></li>
|
||||
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="Hand.html#keypoints" class="tsd-kind-icon">keypoints</a></li>
|
||||
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="Hand.html#landmarks" class="tsd-kind-icon">landmarks</a></li>
|
||||
<li class="tsd-kind-property tsd-parent-kind-interface"><a href="Hand.html#score" class="tsd-kind-icon">score</a></li>
|
||||
</ul>
|
||||
</section>
|
||||
|
|
@ -110,7 +112,7 @@
|
|||
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
|
||||
<a name="annotations" class="tsd-anchor"></a>
|
||||
<h3>annotations</h3>
|
||||
<div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol"><</span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">></span></div>
|
||||
<div class="tsd-signature tsd-kind-icon">annotations<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol"><</span><span class="tsd-signature-type">"index"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"middle"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"pinky"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"ring"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"thumb"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"palm"</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">[</span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-type">number</span><span class="tsd-signature-symbol">]</span><span class="tsd-signature-symbol">[]</span><span class="tsd-signature-symbol">></span></div>
|
||||
<aside class="tsd-sources">
|
||||
</aside>
|
||||
</section>
|
||||
|
|
@ -142,6 +144,13 @@
|
|||
<aside class="tsd-sources">
|
||||
</aside>
|
||||
</section>
|
||||
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
|
||||
<a name="landmarks" class="tsd-anchor"></a>
|
||||
<h3>landmarks</h3>
|
||||
<div class="tsd-signature tsd-kind-icon">landmarks<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Record</span><span class="tsd-signature-symbol"><</span><span class="tsd-signature-type">"index"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"middle"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"pinky"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"ring"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"thumb"</span><span class="tsd-signature-symbol">, </span><span class="tsd-signature-symbol">{ </span>curl<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">"none"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"full"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"half"</span><span class="tsd-signature-symbol">; </span>direction<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">"verticalUp"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"verticalDown"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"horizontalLeft"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"horizontalRight"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"diagonalUpRight"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"diagonalUpLeft"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"diagonalDownRight"</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">"diagonalDownLeft"</span><span class="tsd-signature-symbol"> }</span><span class="tsd-signature-symbol">></span></div>
|
||||
<aside class="tsd-sources">
|
||||
</aside>
|
||||
</section>
|
||||
<section class="tsd-panel tsd-member tsd-kind-property tsd-parent-kind-interface">
|
||||
<a name="score" class="tsd-anchor"></a>
|
||||
<h3>score</h3>
|
||||
|
|
@ -199,6 +208,9 @@
|
|||
<li class=" tsd-kind-property tsd-parent-kind-interface">
|
||||
<a href="Hand.html#keypoints" class="tsd-kind-icon">keypoints</a>
|
||||
</li>
|
||||
<li class=" tsd-kind-property tsd-parent-kind-interface">
|
||||
<a href="Hand.html#landmarks" class="tsd-kind-icon">landmarks</a>
|
||||
</li>
|
||||
<li class=" tsd-kind-property tsd-parent-kind-interface">
|
||||
<a href="Hand.html#score" class="tsd-kind-icon">score</a>
|
||||
</li>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
declare const Finger: {
|
||||
thumb: number;
|
||||
index: number;
|
||||
middle: number;
|
||||
ring: number;
|
||||
pinky: number;
|
||||
all: number[];
|
||||
nameMapping: {
|
||||
0: string;
|
||||
1: string;
|
||||
2: string;
|
||||
3: string;
|
||||
4: string;
|
||||
};
|
||||
pointsMapping: {
|
||||
0: number[][];
|
||||
1: number[][];
|
||||
2: number[][];
|
||||
3: number[][];
|
||||
4: number[][];
|
||||
};
|
||||
getName: (value: any) => any;
|
||||
getPoints: (value: any) => any;
|
||||
};
|
||||
declare const FingerCurl: {
|
||||
none: number;
|
||||
half: number;
|
||||
full: number;
|
||||
nameMapping: {
|
||||
0: string;
|
||||
1: string;
|
||||
2: string;
|
||||
};
|
||||
getName: (value: any) => any;
|
||||
};
|
||||
declare const FingerDirection: {
|
||||
verticalUp: number;
|
||||
verticalDown: number;
|
||||
horizontalLeft: number;
|
||||
horizontalRight: number;
|
||||
diagonalUpRight: number;
|
||||
diagonalUpLeft: number;
|
||||
diagonalDownRight: number;
|
||||
diagonalDownLeft: number;
|
||||
nameMapping: {
|
||||
0: string;
|
||||
1: string;
|
||||
2: string;
|
||||
3: string;
|
||||
4: string;
|
||||
5: string;
|
||||
6: string;
|
||||
7: string;
|
||||
};
|
||||
getName: (value: any) => any;
|
||||
};
|
||||
export { Finger, FingerCurl, FingerDirection };
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export declare function estimate(landmarks: any): {
|
||||
curls: number[];
|
||||
directions: number[];
|
||||
};
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export declare function analyze(keypoints: any): {};
|
||||
export declare function match(keypoints: any): {
|
||||
name: string;
|
||||
confidence: number;
|
||||
}[];
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
export default class Gesture {
|
||||
name: any;
|
||||
curls: any;
|
||||
directions: any;
|
||||
weights: any;
|
||||
weightsRelative: any;
|
||||
constructor(name: any);
|
||||
addCurl(finger: any, curl: any, confidence: any): void;
|
||||
addDirection(finger: any, position: any, confidence: any): void;
|
||||
setWeight(finger: any, weight: any): void;
|
||||
matchAgainst(detectedCurls: any, detectedDirections: any): number;
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import Gesture from './gesture';
|
||||
declare const _default: Gesture[];
|
||||
export default _default;
|
||||
|
|
@ -17,7 +17,7 @@ export declare type BodyGesture = `leaning ${'left' | 'right'}` | `raise ${'left
|
|||
/**
|
||||
* @typedef BodyGesture
|
||||
*/
|
||||
export declare type HandGesture = `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} forward` | `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} up`;
|
||||
export declare type HandGesture = `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} forward` | `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} up` | 'victory' | 'thumbs up';
|
||||
export declare const body: (res: any) => Gesture[];
|
||||
export declare const face: (res: any) => Gesture[];
|
||||
export declare const iris: (res: any) => Gesture[];
|
||||
|
|
|
|||
|
|
@ -96,8 +96,9 @@ export interface Body {
|
|||
* - score: detection confidence score as value
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part
|
||||
* - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part with keypoints
|
||||
* - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
|
||||
*/
|
||||
export interface Hand {
|
||||
id: number;
|
||||
|
|
@ -105,7 +106,11 @@ export interface Hand {
|
|||
box: [number, number, number, number];
|
||||
boxRaw: [number, number, number, number];
|
||||
keypoints: Array<[number, number, number]>;
|
||||
annotations: Record<string, Array<[number, number, number]>>;
|
||||
annotations: Record<'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm', Array<[number, number, number]>>;
|
||||
landmarks: Record<'index' | 'middle' | 'pinky' | 'ring' | 'thumb', {
|
||||
curl: 'none' | 'half' | 'full';
|
||||
direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft';
|
||||
}>;
|
||||
}
|
||||
/** Object results
|
||||
*
|
||||
|
|
|
|||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit c12e036ac382043f4b3a85cf71f93927af56cfe4
|
||||
Subproject commit 65558ea91f6d5ec2dbc46bf9c46c592d34dce706
|
||||
Loading…
Reference in New Issue