implement finger poses in hand detection and gestures

CHANGELOG.md

@@ -9,7 +9,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
-### **HEAD -> main** 2021/08/19 mandic00@live.com
+### **HEAD -> main** 2021/08/20 mandic00@live.com
 
 
 ### **2.1.4** 2021/08/19 mandic00@live.com

demo/index.js

@@ -40,7 +40,7 @@ let userConfig = {
     enabled: false,
     flip: false,
   },
-  face: { enabled: true,
+  face: { enabled: false,
     detector: { return: false, rotation: true },
     mesh: { enabled: true },
     iris: { enabled: true },
@@ -48,7 +48,7 @@ let userConfig = {
     emotion: { enabled: false },
   },
   object: { enabled: false },
-  gesture: { enabled: false },
+  gesture: { enabled: true },
   hand: { enabled: false },
   body: { enabled: false },
   // body: { enabled: true, modelPath: 'movenet-multipose.json' },

src/draw/draw.ts

@@ -414,12 +414,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
         ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
       };
       ctx.font = localOptions.font;
-      addHandLabel(h.annotations['indexFinger'], 'index');
+      addHandLabel(h.annotations['index'], 'index');
-      addHandLabel(h.annotations['middleFinger'], 'middle');
+      addHandLabel(h.annotations['middle'], 'middle');
-      addHandLabel(h.annotations['ringFinger'], 'ring');
+      addHandLabel(h.annotations['ring'], 'ring');
       addHandLabel(h.annotations['pinky'], 'pinky');
       addHandLabel(h.annotations['thumb'], 'thumb');
-      addHandLabel(h.annotations['palmBase'], 'palm');
+      addHandLabel(h.annotations['palm'], 'palm');
     }
     if (localOptions.drawPolygons) {
       const addHandLine = (part) => {
@@ -433,12 +433,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, dra
         }
       };
       ctx.lineWidth = localOptions.lineWidth;
-      addHandLine(h.annotations['indexFinger']);
+      addHandLine(h.annotations['index']);
-      addHandLine(h.annotations['middleFinger']);
+      addHandLine(h.annotations['middle']);
-      addHandLine(h.annotations['ringFinger']);
+      addHandLine(h.annotations['ring']);
       addHandLine(h.annotations['pinky']);
       addHandLine(h.annotations['thumb']);
-      // addPart(h.annotations.palmBase);
+      // addPart(h.annotations.palm);
     }
   }
 }

src/fingerpose/estimator.ts

@@ -0,0 +1,205 @@
+import { Finger, FingerCurl, FingerDirection } from './description';
+
+const options = {
+  // curl estimation
+  HALF_CURL_START_LIMIT: 60.0,
+  NO_CURL_START_LIMIT: 130.0,
+  // direction estimation
+  DISTANCE_VOTE_POWER: 1.1,
+  SINGLE_ANGLE_VOTE_POWER: 0.9,
+  TOTAL_ANGLE_VOTE_POWER: 1.6,
+};
+
+function calculateSlope(point1x, point1y, point2x, point2y) {
+  const value = (point1y - point2y) / (point1x - point2x);
+  let slope = Math.atan(value) * 180 / Math.PI;
+  if (slope <= 0) slope = -slope;
+  else if (slope > 0) slope = 180 - slope;
+  return slope;
+}
+
+// point1, point2 are 2d or 3d point arrays (xy[z])
+// returns either a single scalar (2d) or array of two slopes (3d)
+function getSlopes(point1, point2) {
+  const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]);
+  if (point1.length === 2) return slopeXY;
+  const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]);
+  return [slopeXY, slopeYZ];
+}
+
+function angleOrientationAt(angle, weightageAt = 1.0) {
+  let isVertical = 0;
+  let isDiagonal = 0;
+  let isHorizontal = 0;
+  if (angle >= 75.0 && angle <= 105.0) isVertical = 1 * weightageAt;
+  else if (angle >= 25.0 && angle <= 155.0) isDiagonal = 1 * weightageAt;
+  else isHorizontal = 1 * weightageAt;
+  return [isVertical, isDiagonal, isHorizontal];
+}
+
+function estimateFingerCurl(startPoint, midPoint, endPoint) {
+  const start_mid_x_dist = startPoint[0] - midPoint[0];
+  const start_end_x_dist = startPoint[0] - endPoint[0];
+  const mid_end_x_dist = midPoint[0] - endPoint[0];
+  const start_mid_y_dist = startPoint[1] - midPoint[1];
+  const start_end_y_dist = startPoint[1] - endPoint[1];
+  const mid_end_y_dist = midPoint[1] - endPoint[1];
+  const start_mid_z_dist = startPoint[2] - midPoint[2];
+  const start_end_z_dist = startPoint[2] - endPoint[2];
+  const mid_end_z_dist = midPoint[2] - endPoint[2];
+  const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist + start_mid_z_dist * start_mid_z_dist);
+  const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist + start_end_z_dist * start_end_z_dist);
+  const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist + mid_end_z_dist * mid_end_z_dist);
+  let cos_in = (mid_end_dist * mid_end_dist + start_mid_dist * start_mid_dist - start_end_dist * start_end_dist) / (2 * mid_end_dist * start_mid_dist);
+  if (cos_in > 1.0) cos_in = 1.0;
+  else if (cos_in < -1.0) cos_in = -1.0;
+  let angleOfCurve = Math.acos(cos_in);
+  angleOfCurve = (57.2958 * angleOfCurve) % 180;
+  let fingerCurl;
+  if (angleOfCurve > options.NO_CURL_START_LIMIT) fingerCurl = FingerCurl.none;
+  else if (angleOfCurve > options.HALF_CURL_START_LIMIT) fingerCurl = FingerCurl.half;
+  else fingerCurl = FingerCurl.full;
+  return fingerCurl;
+}
+
+function estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
+  let estimatedDirection;
+  if (max_dist_x === Math.abs(start_end_x_dist)) {
+    if (start_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
+    else estimatedDirection = FingerDirection.horizontalRight;
+  } else if (max_dist_x === Math.abs(start_mid_x_dist)) {
+    if (start_mid_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
+    else estimatedDirection = FingerDirection.horizontalRight;
+  } else {
+    if (mid_end_x_dist > 0) estimatedDirection = FingerDirection.horizontalLeft;
+    else estimatedDirection = FingerDirection.horizontalRight;
+  }
+  return estimatedDirection;
+}
+
+function estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y) {
+  let estimatedDirection;
+  if (max_dist_y === Math.abs(start_end_y_dist)) {
+    if (start_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
+    else estimatedDirection = FingerDirection.verticalUp;
+  } else if (max_dist_y === Math.abs(start_mid_y_dist)) {
+    if (start_mid_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
+    else estimatedDirection = FingerDirection.verticalUp;
+  } else {
+    if (mid_end_y_dist < 0) estimatedDirection = FingerDirection.verticalDown;
+    else estimatedDirection = FingerDirection.verticalUp;
+  }
+  return estimatedDirection;
+}
+
+function estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x) {
+  let estimatedDirection;
+  const reqd_vertical_direction = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
+  const reqd_horizontal_direction = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
+  if (reqd_vertical_direction === FingerDirection.verticalUp) {
+    if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalUpLeft;
+    else estimatedDirection = FingerDirection.diagonalUpRight;
+  } else {
+    if (reqd_horizontal_direction === FingerDirection.horizontalLeft) estimatedDirection = FingerDirection.diagonalDownLeft;
+    else estimatedDirection = FingerDirection.diagonalDownRight;
+  }
+  return estimatedDirection;
+}
+
+function calculateFingerDirection(startPoint, midPoint, endPoint, fingerSlopes) {
+  const start_mid_x_dist = startPoint[0] - midPoint[0];
+  const start_end_x_dist = startPoint[0] - endPoint[0];
+  const mid_end_x_dist = midPoint[0] - endPoint[0];
+  const start_mid_y_dist = startPoint[1] - midPoint[1];
+  const start_end_y_dist = startPoint[1] - endPoint[1];
+  const mid_end_y_dist = midPoint[1] - endPoint[1];
+  const max_dist_x = Math.max(Math.abs(start_mid_x_dist), Math.abs(start_end_x_dist), Math.abs(mid_end_x_dist));
+  const max_dist_y = Math.max(Math.abs(start_mid_y_dist), Math.abs(start_end_y_dist), Math.abs(mid_end_y_dist));
+  let voteVertical = 0.0;
+  let voteDiagonal = 0.0;
+  let voteHorizontal = 0.0;
+  const start_end_x_y_dist_ratio = max_dist_y / (max_dist_x + 0.00001);
+  if (start_end_x_y_dist_ratio > 1.5) voteVertical += options.DISTANCE_VOTE_POWER;
+  else if (start_end_x_y_dist_ratio > 0.66) voteDiagonal += options.DISTANCE_VOTE_POWER;
+  else voteHorizontal += options.DISTANCE_VOTE_POWER;
+  const start_mid_dist = Math.sqrt(start_mid_x_dist * start_mid_x_dist + start_mid_y_dist * start_mid_y_dist);
+  const start_end_dist = Math.sqrt(start_end_x_dist * start_end_x_dist + start_end_y_dist * start_end_y_dist);
+  const mid_end_dist = Math.sqrt(mid_end_x_dist * mid_end_x_dist + mid_end_y_dist * mid_end_y_dist);
+  const max_dist = Math.max(start_mid_dist, start_end_dist, mid_end_dist);
+  let calc_start_point_x = startPoint[0];
+  let calc_start_point_y = startPoint[1];
+  let calc_end_point_x = endPoint[0];
+  let calc_end_point_y = endPoint[1];
+  if (max_dist === start_mid_dist) {
+    calc_end_point_x = endPoint[0];
+    calc_end_point_y = endPoint[1];
+  } else if (max_dist === mid_end_dist) {
+    calc_start_point_x = midPoint[0];
+    calc_start_point_y = midPoint[1];
+  }
+  const calcStartPoint = [calc_start_point_x, calc_start_point_y];
+  const calcEndPoint = [calc_end_point_x, calc_end_point_y];
+  const totalAngle = getSlopes(calcStartPoint, calcEndPoint);
+  const votes = angleOrientationAt(totalAngle, options.TOTAL_ANGLE_VOTE_POWER);
+  voteVertical += votes[0];
+  voteDiagonal += votes[1];
+  voteHorizontal += votes[2];
+  for (const fingerSlope of fingerSlopes) {
+    const fingerVotes = angleOrientationAt(fingerSlope, options.SINGLE_ANGLE_VOTE_POWER);
+    voteVertical += fingerVotes[0];
+    voteDiagonal += fingerVotes[1];
+    voteHorizontal += fingerVotes[2];
+  }
+  // in case of tie, highest preference goes to Vertical,
+  // followed by horizontal and then diagonal
+  let estimatedDirection;
+  if (voteVertical === Math.max(voteVertical, voteDiagonal, voteHorizontal)) {
+    estimatedDirection = estimateVerticalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y);
+  } else if (voteHorizontal === Math.max(voteDiagonal, voteHorizontal)) {
+    estimatedDirection = estimateHorizontalDirection(start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
+  } else {
+    estimatedDirection = estimateDiagonalDirection(start_end_y_dist, start_mid_y_dist, mid_end_y_dist, max_dist_y, start_end_x_dist, start_mid_x_dist, mid_end_x_dist, max_dist_x);
+  }
+  return estimatedDirection;
+}
+
+export function estimate(landmarks) {
+  // step 1: calculate slopes
+  const slopesXY: Array<number[]> = [];
+  const slopesYZ: Array<number[]> = [];
+  for (const finger of Finger.all) {
+    const points = Finger.getPoints(finger);
+    const slopeAtXY: Array<number> = [];
+    const slopeAtYZ: Array<number> = [];
+    for (const point of points) {
+      const point1 = landmarks[point[0]];
+      const point2 = landmarks[point[1]];
+      // calculate single slope
+      const slopes = getSlopes(point1, point2);
+      const slopeXY = slopes[0];
+      const slopeYZ = slopes[1];
+      slopeAtXY.push(slopeXY);
+      slopeAtYZ.push(slopeYZ);
+    }
+    slopesXY.push(slopeAtXY);
+    slopesYZ.push(slopeAtYZ);
+  }
+
+  // step 2: calculate orientations
+  const fingerCurls: Array<number> = [];
+  const fingerDirections: Array<number> = [];
+  for (const finger of Finger.all) {
+    // start finger predictions from palm - except for thumb
+    const pointIndexAt = (finger === Finger.thumb) ? 1 : 0;
+    const fingerPointsAt = Finger.getPoints(finger);
+    const startPoint = landmarks[fingerPointsAt[pointIndexAt][0]];
+    const midPoint = landmarks[fingerPointsAt[pointIndexAt + 1][1]];
+    const endPoint = landmarks[fingerPointsAt[3][1]];
+    // check if finger is curled
+    const fingerCurled = estimateFingerCurl(startPoint, midPoint, endPoint);
+    const fingerPosition = calculateFingerDirection(startPoint, midPoint, endPoint, slopesXY[finger].slice(pointIndexAt));
+    fingerCurls[finger] = fingerCurled;
+    fingerDirections[finger] = fingerPosition;
+  }
+  return { curls: fingerCurls, directions: fingerDirections };
+}

src/fingerpose/gesture.ts

@@ -0,0 +1,75 @@
+export default class Gesture {
+  name;
+  curls;
+  directions;
+  weights;
+  weightsRelative;
+
+  constructor(name) {
+    // name (should be unique)
+    this.name = name;
+    this.curls = {};
+    this.directions = {};
+    this.weights = [1.0, 1.0, 1.0, 1.0, 1.0];
+    this.weightsRelative = [1.0, 1.0, 1.0, 1.0, 1.0];
+  }
+
+  addCurl(finger, curl, confidence) {
+    if (typeof this.curls[finger] === 'undefined') this.curls[finger] = [];
+    this.curls[finger].push([curl, confidence]);
+  }
+
+  addDirection(finger, position, confidence) {
+    if (!this.directions[finger]) this.directions[finger] = [];
+    this.directions[finger].push([position, confidence]);
+  }
+
+  setWeight(finger, weight) {
+    this.weights[finger] = weight;
+    // recalculate relative weights
+    const total = this.weights.reduce((a, b) => a + b, 0);
+    this.weightsRelative = this.weights.map((el) => el * 5 / total);
+  }
+
+  matchAgainst(detectedCurls, detectedDirections) {
+    let confidence = 0.0;
+    // look at the detected curl of each finger and compare with
+    // the expected curl of this finger inside current gesture
+    for (const fingerIdx in detectedCurls) {
+      const detectedCurl = detectedCurls[fingerIdx];
+      const expectedCurls = this.curls[fingerIdx];
+      if (typeof expectedCurls === 'undefined') {
+        // no curl description available for this finger
+        // add default confidence of "1"
+        confidence += this.weightsRelative[fingerIdx];
+        continue;
+      }
+      // compare to each possible curl of this specific finger
+      for (const [expectedCurl, score] of expectedCurls) {
+        if (detectedCurl === expectedCurl) {
+          confidence += score * this.weightsRelative[fingerIdx];
+          break;
+        }
+      }
+    }
+    // same for detected direction of each finger
+    for (const fingerIdx in detectedDirections) {
+      const detectedDirection = detectedDirections[fingerIdx];
+      const expectedDirections = this.directions[fingerIdx];
+      if (typeof expectedDirections === 'undefined') {
+        // no direction description available for this finger
+        // add default confidence of "1"
+        confidence += this.weightsRelative[fingerIdx];
+        continue;
+      }
+      // compare to each possible direction of this specific finger
+      for (const [expectedDirection, score] of expectedDirections) {
+        if (detectedDirection === expectedDirection) {
+          confidence += score * this.weightsRelative[fingerIdx];
+          break;
+        }
+      }
+    }
+    return confidence / 10;
+  }
+}

src/fingerpose/gestures.ts

@@ -0,0 +1,39 @@
+import { Finger, FingerCurl, FingerDirection } from './description';
+import Gesture from './gesture';
+
+// describe thumbs up gesture 👍
+const ThumbsUp = new Gesture('thumbs up');
+ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1.0);
+ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
+ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
+ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
+for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
+  ThumbsUp.addCurl(finger, FingerCurl.full, 1.0);
+  ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1.0);
+  ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1.0);
+}
+
+// describe Victory gesture ✌️
+const Victory = new Gesture('victory');
+Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
+Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
+Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
+Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1.0);
+Victory.addCurl(Finger.index, FingerCurl.none, 1.0);
+Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
+Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1.0);
+Victory.addCurl(Finger.middle, FingerCurl.none, 1.0);
+Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1.0);
+Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
+Victory.addCurl(Finger.ring, FingerCurl.full, 1.0);
+Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
+Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1.0);
+Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
+Victory.addCurl(Finger.pinky, FingerCurl.full, 1.0);
+Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
+Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1.0);
+Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
+Victory.setWeight(Finger.index, 2);
+Victory.setWeight(Finger.middle, 2);
+
+export default [ThumbsUp, Victory];

src/gesture/gesture.ts

@@ -3,6 +3,7 @@
  */
 
 import { Gesture } from '../result';
+import * as fingerPose from '../fingerpose/fingerpose';
 
 /**
  * @typedef FaceGesture
@@ -33,8 +34,10 @@ export type BodyGesture =
  * @typedef BodyGesture
  */
 export type HandGesture =
-  `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} forward`
+  `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} forward`
-  | `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} up`;
+  | `${'thumb' | 'index' | 'middle' | 'ring' | 'pinky'} up`
+  | 'victory'
+  | 'thumbs up';
 
 export const body = (res): Gesture[] => {
   if (!res) return [];
@@ -129,6 +132,8 @@ export const hand = (res): Gesture[] => {
       const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
       gestures.push({ hand: i, gesture: `${highest.name} up` as HandGesture });
     }
+    const poses = fingerPose.match(res[i]['keypoints']);
+    for (const pose of poses) gestures.push({ hand: i, gesture: pose.name as HandGesture });
   }
   return gestures;
 };

src/handpose/handpose.ts

@@ -6,17 +6,18 @@ import { log, join } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as handdetector from './handdetector';
 import * as handpipeline from './handpipeline';
+import * as fingerPose from '../fingerpose/fingerpose';
 import { Hand } from '../result';
 import { Tensor, GraphModel } from '../tfjs/types';
 import { Config } from '../config';
 
 const meshAnnotations = {
   thumb: [1, 2, 3, 4],
-  indexFinger: [5, 6, 7, 8],
+  index: [5, 6, 7, 8],
-  middleFinger: [9, 10, 11, 12],
+  middle: [9, 10, 11, 12],
-  ringFinger: [13, 14, 15, 16],
+  ring: [13, 14, 15, 16],
   pinky: [17, 18, 19, 20],
-  palmBase: [0],
+  palm: [0],
 };
 
 let handDetectorModel: GraphModel | null;
@@ -64,7 +65,16 @@ export async function predict(input: Tensor, config: Config): Promise<Hand[]> {
         (predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / (input.shape[1] || 0),
       ];
     }
-    hands.push({ id: i, score: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, keypoints, annotations });
+    const landmarks = fingerPose.analyze(keypoints);
+    hands.push({
+      id: i,
+      score: Math.round(100 * predictions[i].confidence) / 100,
+      box,
+      boxRaw,
+      keypoints,
+      annotations: annotations as Hand['annotations'],
+      landmarks: landmarks as Hand['landmarks'],
+    });
   }
   return hands;
 }

src/interpolate.ts

@@ -68,7 +68,7 @@ export function calc(newResult: Result): Result {
         annotations[key] = newResult.hand[i].annotations[key]
           .map((val, j) => val.map((coord, k) => ((bufferedFactor - 1) * bufferedResult.hand[i].annotations[key][j][k] + coord) / bufferedFactor));
       }
-      bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations }; // shallow clone plus updated values
+      bufferedResult.hand[i] = { ...newResult.hand[i], box, boxRaw, keypoints, annotations: annotations as Hand['annotations'] }; // shallow clone plus updated values
     }
   }
 

src/result.ts

@@ -90,8 +90,9 @@ export interface Body {
  * - score: detection confidence score as value
  * - box: bounding box: x, y, width, height normalized to input image resolution
  * - boxRaw: bounding box: x, y, width, height normalized to 0..1
- * - landmarks: landmarks as array of [x, y, z] points of hand, normalized to image resolution
+ * - keypoints: keypoints as array of [x, y, z] points of hand, normalized to image resolution
- * - annotations: annotated landmarks for each hand part
+ * - annotations: annotated landmarks for each hand part with keypoints
+ * - landmarks: annotated landmarks for eachb hand part with logical curl and direction strings
  */
 export interface Hand {
   id: number,
@@ -99,7 +100,14 @@ export interface Hand {
   box: [number, number, number, number],
   boxRaw: [number, number, number, number],
   keypoints: Array<[number, number, number]>,
-  annotations: Record<string, Array<[number, number, number]>>,
+  annotations: Record<
+    'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
+    Array<[number, number, number]>
+  >,
+  landmarks: Record<
+    'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
+    { curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
+  >,
 }
 
 /** Object results

wiki

@@ -1 +1 @@
-Subproject commit c12e036ac382043f4b3a85cf71f93927af56cfe4
+Subproject commit 65558ea91f6d5ec2dbc46bf9c46c592d34dce706