mirror of https://github.com/vladmandic/human
implement finger poses in hand detection and gestures
Vladimir Mandic
parent
54d717bbff
commit
ac83b3d153
|
|
@ -9,7 +9,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/08/19 mandic00@live.com
|
||||
### **HEAD -> main** 2021/08/20 mandic00@live.com
|
||||
|
||||
|
||||
### **2.1.4** 2021/08/19 mandic00@live.com
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ let userConfig = {
|
|||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: true,
|
||||
face: { enabled: false,
|
||||
detector: { return: false, rotation: true },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: true },
|
||||
|
|
@ -48,7 +48,7 @@ let userConfig = {
|
|||
emotion: { enabled: false },
|
||||
},
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
hand: { enabled: false },
|
||||
body: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'movenet-multipose.json' },
|
||||
|
|
|
|||
|
|
@ -414,12 +414,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations['indexFinger'], 'index');
|
||||
addHandLabel(h.annotations['middleFinger'], 'middle');
|
||||
addHandLabel(h.annotations['ringFinger'], 'ring');
|
||||
addHandLabel(h.annotations['index'], 'index');
|
||||
addHandLabel(h.annotations['middle'], 'middle');
|
||||
addHandLabel(h.annotations['ring'], 'ring');
|
||||
addHandLabel(h.annotations['pinky'], 'pinky');
|
||||
addHandLabel(h.annotations['thumb'], 'thumb');
|
||||
addHandLabel(h.annotations['palmBase'], 'palm');
|
||||
addHandLabel(h.annotations['palm'], 'palm');
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -433,12 +433,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations['indexFinger']);
|
||||
addHandLine(h.annotations['middleFinger']);
|
||||
addHandLine(h.annotations['ringFinger']);
|
||||
addHandLine(h.annotations['index']);
|
||||
addHandLine(h.annotations['middle']);
|
||||
addHandLine(h.annotations['ring']);
|
||||
addHandLine(h.annotations['pinky']);
|
||||
addHandLine(h.annotations['thumb']);
|
||||
// addPart(h.annotations.palmBase);
|
||||
// addPart(h.annotations.palm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,205 @@
|
|||
import { Finger, FingerCurl, FingerDirection } from './description';
|
||||
|
||||
const options = {
|
||||
// curl estimation
|
||||
HALF_CURL_START_LIMIT: 60.0,
|
||||
NO_CURL_START_LIMIT: 130.0,
|
||||
// direction estimation
|
||||
DISTANCE_VOTE_POWER: 1.1,
|
||||
SINGLE_ANGLE_VOTE_POWER: 0.9,
|
||||
TOTAL_ANGLE_VOTE_POWER: 1.6,
|
||||
};
|
||||
|
||||
function calculateSlope(point1x, point1y, point2x, point2y) {
|
||||
const value = (point1y - point2y) / (point1x - point2x);
|
||||
let slope = Math.atan(value) * 180 / Math.PI;
|
||||
if (slope <= 0) slope = -slope;
|
||||
else if (slope > 0) slope = 180 - slope;
|
||||
return slope;
|
||||
}
|
||||
|
||||
// point1, point2 are 2d or 3d point arrays (xy[z])
|
||||
// returns either a single scalar (2d) or array of two slopes (3d)
|
||||
function getSlopes(point1, point2) {
|
||||
const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
|
||||
if (point1.length === 2) return slopeXY;
|
||||
const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
|
||||
return [slopeXY, slopeYZ];
|
||||
}
|
||||
|
||||
function angleOrientationAt(angle, weightageAt = 1.0) {
|
||||
let isVertical = 0;
|
||||
let isDiagonal = 0;
|
||||
let isHorizontal = 0;
|
||||
if (angle >= 75.0 && angle <= 105.0) isVertical = 1 * weightageAt;
|
||||
else if (angle >= 25.0 && angle <= 155.0) isDiagonal = 1 * weightageAt;
|
||||
else isHorizontal = 1 * weightageAt;
|
||||
return [isVertical, isDiagonal, isHorizontal];
|
||||
}
|
||||
|
||||
function estimateFingerCurl(startPoint, midPoint, endPoint) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const start_mid_z_dist = startPoint[2] - midPoint[2];
|
||||
const start_end_z_dist = startPoint[2] - endPoint[2];
|
||||
const mid_end_z_dist = midPoint[2] - endPoint[2];
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
|
||||
let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
|
||||
if (cos_in > 1.0) cos_in = 1.0;
|
||||
else if (cos_in < -1.0) cos_in = -1.0;
|
||||
let angleOfCurve = Math.acos(cos_in);
|
||||
angleOfCurve = (57.2958 * angleOfCurve) % 180;
|
||||
let fingerCurl;
|
||||
if (angleOfCurve > options.NO_CURL_START_LIMIT) fingerCurl = FingerCurl.none;
|
||||
else if (angleOfCurve > options.HALF_CURL_START_LIMIT) fingerCurl = FingerCurl.half;
|
||||
else fingerCurl = FingerCurl.full;
|
||||
return fingerCurl;
|
||||
}
|
||||
|
||||
function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_x === Math.abs(start_end_x_dist)) {
|
||||
if (start_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else if (max_dist_x === Math.abs(start_mid_x_dist)) {
|
||||
if (start_mid_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
} else {
|
||||
if (mid_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
|
||||
else estimatedDirection = FingerDirection.horizontalRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
|
||||
let estimatedDirection;
|
||||
if (max_dist_y === Math.abs(start_end_y_dist)) {
|
||||
if (start_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
} else if (max_dist_y === Math.abs(start_mid_y_dist)) {
|
||||
if (start_mid_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
} else {
|
||||
if (mid_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
|
||||
else estimatedDirection = FingerDirection.verticalUp;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
|
||||
let estimatedDirection;
|
||||
const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
if (reqd_vertical_direction === FingerDirection.verticalUp) {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalUpLeft;
|
||||
else estimatedDirection = FingerDirection.diagonalUpRight;
|
||||
} else {
|
||||
if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalDownLeft;
|
||||
else estimatedDirection = FingerDirection.diagonalDownRight;
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
|
||||
const start_mid_x_dist = startPoint[0] - midPoint[0];
|
||||
const start_end_x_dist = startPoint[0] - endPoint[0];
|
||||
const mid_end_x_dist = midPoint[0] - endPoint[0];
|
||||
const start_mid_y_dist = startPoint[1] - midPoint[1];
|
||||
const start_end_y_dist = startPoint[1] - endPoint[1];
|
||||
const mid_end_y_dist = midPoint[1] - endPoint[1];
|
||||
const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
|
||||
const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
|
||||
let voteVertical = 0.0;
|
||||
let voteDiagonal = 0.0;
|
||||
let voteHorizontal = 0.0;
|
||||
const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 0.00001);
|
||||
if (start_end_x_y_dist_ratio > 1.5) voteVertical += options.DISTANCE_VOTE_POWER;
|
||||
else if (start_end_x_y_dist_ratio > 0.66) voteDiagonal += options.DISTANCE_VOTE_POWER;
|
||||
else voteHorizontal += options.DISTANCE_VOTE_POWER;
|
||||
const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
|
||||
const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
|
||||
const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
|
||||
const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
|
||||
let calc_start_point_x = startPoint[0];
|
||||
let calc_start_point_y = startPoint[1];
|
||||
let calc_end_point_x = endPoint[0];
|
||||
let calc_end_point_y = endPoint[1];
|
||||
if (max_dist === start_mid_dist) {
|
||||
calc_end_point_x = endPoint[0];
|
||||
calc_end_point_y = endPoint[1];
|
||||
} else if (max_dist === mid_end_dist) {
|
||||
calc_start_point_x = midPoint[0];
|
||||
calc_start_point_y = midPoint[1];
|
||||
}
|
||||
const calcStartPoint = [calc_start_point_x, calc_start_point_y];
|
||||
const calcEndPoint = [calc_end_point_x, calc_end_point_y];
|
||||
const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
|
||||
const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
|
||||
voteVertical += votes[0];
|
||||
voteDiagonal += votes[1];
|
||||
voteHorizontal += votes[2];
|
||||
for (const fingerSlope of fingerSlopes) {
|
||||
const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
|
||||
voteVertical += fingerVotes[0];
|
||||
voteDiagonal += fingerVotes[1];
|
||||
voteHorizontal += fingerVotes[2];
|
||||
}
|
||||
// in case of tie, highest preference goes to Vertical,
|
||||
// followed by horizontal and then diagonal
|
||||
let estimatedDirection;
|
||||
if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
|
||||
} else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
|
||||
estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
} else {
|
||||
estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
|
||||
}
|
||||
return estimatedDirection;
|
||||
}
|
||||
|
||||
export function estimate(landmarks) {
|
||||
// step 1: calculate slopes
|
||||
const slopesXY: Array<number[]> = [];
|
||||
const slopesYZ: Array<number[]> = [];
|
||||
for (const finger of Finger.all) {
|
||||
const points = Finger.getPoints(finger);
|
||||
const slopeAtXY: Array<number> = [];
|
||||
const slopeAtYZ: Array<number> = [];
|
||||
for (const point of points) {
|
||||
const point1 = landmarks[point[0]];
|
||||
const point2 = landmarks[point[1]];
|
||||
// calculate single slope
|
||||
const slopes = getSlopes(point1, point2);
|
||||
const slopeXY = slopes[0];
|
||||
const slopeYZ = slopes[1];
|
||||
slopeAtXY.push(slopeXY);
|
||||
slopeAtYZ.push(slopeYZ);
|
||||
}
|
||||
slopesXY.push(slopeAtXY);
|
||||
slopesYZ.push(slopeAtYZ);
|
||||
}
|
||||
|
||||
// step 2: calculate orientations
|
||||
const fingerCurls: Array<number> = [];
|
||||
const fingerDirections: Array<number> = [];
|
||||
for (const finger of Finger.all) {
|
||||
// start finger predictions from palm - except for thumb
|
||||
const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;
|
||||
const fingerPointsAt = Finger.getPoints(finger);
|
||||
const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
|
||||
const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
|
||||
const endPoint = landmarks[fingerPointsAt[3][1]];
|
||||
// check if finger is curled
|
||||
const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
|
||||
const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
|
||||
fingerCurls[finger] = fingerCurled;
|
||||
fingerDirections[finger] = fingerPosition;
|
||||
}
|
||||
return { curls: fingerCurls, directions: fingerDirections };
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
export default class Gesture {
|
||||
name;
|
||||
curls;
|
||||
directions;
|
||||
weights;
|
||||
weightsRelative;
|
||||
|
||||
constructor(name) {
|
||||
// name (should be unique)
|
||||
this.name = name;
|
||||
this.curls = {};
|
||||
this.directions = {};
|
||||
this.weights = [1.0, 1.0, 1.0, 1.0, 1.0];
|
||||
this.weightsRelative = [1.0, 1.0, 1.0, 1.0, 1.0];
|
||||
}
|
||||
|
||||
addCurl(finger, curl, confidence) {
|
||||
if (typeof this.curls[finger] === 'undefined') this.curls[finger] = [];
|
||||
this.curls[finger].push([curl, confidence]);
|
||||
}
|
||||
|
||||
addDirection(finger, position, confidence) {
|
||||
if (!this.directions[finger]) this.directions[finger] = [];
|
||||
this.directions[finger].push([position, confidence]);
|
||||
}
|
||||
|
||||
setWeight(finger, weight) {
|
||||
this.weights[finger] = weight;
|
||||
// recalculate relative weights
|
||||
const total = this.weights.reduce((a, b) => a + b, 0);
|
||||
this.weightsRelative = this.weights.map((el) => el * 5 / total);
|
||||
}
|
||||
|
||||
matchAgainst(detectedCurls, detectedDirections) {
|
||||
let confidence = 0.0;
|
||||
// look at the detected curl of each finger and compare with
|
||||
// the expected curl of this finger inside current gesture
|
||||
for (const fingerIdx in detectedCurls) {
|
||||
const detectedCurl = detectedCurls[fingerIdx];
|
||||
const expectedCurls = this.curls[fingerIdx];
|
||||
if (typeof expectedCurls === 'undefined') {
|
||||
// no curl description available for this finger
|
||||
// add default confidence of "1"
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
// compare to each possible curl of this specific finger
|
||||
for (const [expectedCurl, score] of expectedCurls) {
|
||||
if (detectedCurl === expectedCurl) {
|
||||
confidence += score * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// same for detected direction of each finger
|
||||
for (const fingerIdx in detectedDirections) {
|
||||
const detectedDirection = detectedDirections[fingerIdx];
|
||||
const expectedDirections = this.directions[fingerIdx];
|
||||
if (typeof expectedDirections === 'undefined') {
|
||||
// no direction description available for this finger
|
||||
// add default confidence of "1"
|
||||
confidence += this.weightsRelative[fingerIdx];
|
||||
continue;
|
||||
}
|
||||
// compare to each possible direction of this specific finger
|
||||
for (const [expectedDirection, score] of expectedDirections) {
|
||||
if (detectedDirection === expectedDirection) {
|
||||
confidence += score * this.weightsRelative[fingerIdx];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return confidence / 10;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import { Finger, FingerCurl, FingerDirection } from './description';
|
||||
import Gesture from './gesture';
|
||||
|
||||
// describe thumbs up gesture 👍
|
||||
const ThumbsUp = new Gesture('thumbs up');
|
||||
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1.0);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
|
||||
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
|
||||
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
|
||||
ThumbsUp.addCurl(finger, FingerCurl.full, 1.0);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1.0);
|
||||
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1.0);
|
||||
}
|
||||
|
||||
// describe Victory gesture ✌️
|
||||
const Victory = new Gesture('victory');
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
|
||||
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
|
||||
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addCurl(Finger.index, FingerCurl.none, 1.0);
|
||||
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
|
||||
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addCurl(Finger.middle, FingerCurl.none, 1.0);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1.0);
|
||||
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
|
||||
Victory.addCurl(Finger.ring, FingerCurl.full, 1.0);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.addCurl(Finger.pinky, FingerCurl.full, 1.0);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1.0);
|
||||
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
|
||||
Victory.setWeight(Finger.index, 2);
|
||||
Victory.setWeight(Finger.middle, 2);
|
||||
|
||||
export default [ThumbsUp, Victory];
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
*/
|
||||
|
||||
import { Gesture } from '../result';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
|
||||
/**
|
||||
* @typedef FaceGesture
|
||||
|
|
@ -33,8 +34,10 @@ export type BodyGesture =
|
|||
* @typedef BodyGesture
|
||||
*/
|
||||
export type HandGesture =
|
||||
`${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} forward`
|
||||
| `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} up`;
|
||||
`${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} forward`
|
||||
| `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} up`
|
||||
| 'victory'
|
||||
| 'thumbs up';
|
||||
|
||||
export const body = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
|
|
@ -129,6 +132,8 @@ export const hand = (res): Gesture[] => {
|
|||
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
|
||||
gestures.push({ hand: i, gesture: `${highest.name} up` as HandGesture });
|
||||
}
|
||||
const poses = fingerPose.match(res[i]['keypoints']);
|
||||
for (const pose of poses) gestures.push({ hand: i, gesture: pose.name as HandGesture });
|
||||
}
|
||||
return gestures;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,17 +6,18 @@ import { log, join } from '../helpers';
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as handdetector from './handdetector';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
import { Hand } from '../result';
|
||||
import { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { Config } from '../config';
|
||||
|
||||
const meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
indexFinger: [5, 6, 7, 8],
|
||||
middleFinger: [9, 10, 11, 12],
|
||||
ringFinger: [13, 14, 15, 16],
|
||||
index: [5, 6, 7, 8],
|
||||
middle: [9, 10, 11, 12],
|
||||
ring: [13, 14, 15, 16],
|
||||
pinky: [17, 18, 19, 20],
|
||||
palmBase: [0],
|
||||
palm: [0],
|
||||
};
|
||||
|
||||
let handDetectorModel: GraphModel | null;
|
||||
|
|
@ -64,7 +65,16 @@ export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
|
|||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0),
|
||||
];
|
||||
}
|
||||
hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations });
|
||||
const landmarks = fingerPose.analyze(keypoints);
|
||||
hands.push({
|
||||
id: i,
|
||||
score: Math.round(100 * predictions[i].confidence) / 100,
|
||||
box,
|
||||
boxRaw,
|
||||
keypoints,
|
||||
annotations: annotations as Hand['annotations'],
|
||||
landmarks: landmarks as Hand['landmarks'],
|
||||
});
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ export function calc(newResult: Result): Result {
|
|||
annotations[key] = newResult.hand[i].annotations[key]
|
||||
.map((val, j) => val.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor));
|
||||
}
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations }; // shallow clone plus updated values
|
||||
bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as Hand['annotations'] }; // shallow clone plus updated values
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -90,8 +90,9 @@ export interface Body {
|
|||
* - score: detection confidence score as value
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part
|
||||
* - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations: annotated landmarks for each hand part with keypoints
|
||||
* - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
|
||||
*/
|
||||
export interface Hand {
|
||||
id: number,
|
||||
|
|
@ -99,7 +100,14 @@ export interface Hand {
|
|||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
keypoints: Array<[number, number, number]>,
|
||||
annotations: Record<string, Array<[number, number, number]>>,
|
||||
annotations: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
|
||||
Array<[number, number, number]>
|
||||
>,
|
||||
landmarks: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
|
||||
{ curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
|
||||
>,
|
||||
}
|
||||
|
||||
/** Object results
|
||||
|
|
|
|||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit c12e036ac382043f4b3a85cf71f93927af56cfe4
|
||||
Subproject commit 65558ea91f6d5ec2dbc46bf9c46c592d34dce706
|
||||
Loading…
Reference in New Issue