strong type for string enums

CHANGELOG.md

@@ -9,8 +9,9 @@
   
 ## Changelog
   
-### **HEAD -> main** 2021/12/13 mandic00@live.com
+### **HEAD -> main** 2021/12/14 mandic00@live.com
 
+- rebuild
 - fix node detection in electron environment
 
 ### **2.5.5** 2021/12/01 mandic00@live.com

TODO.md

@@ -10,7 +10,6 @@
 - Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE
 - TFLite models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
 - Body segmentation: `robust-video-matting`
-- TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
 
 <br><hr><br>
 
@@ -57,3 +56,4 @@ Other:
 - Fix face detect box scale and rotation
 - Fix body interpolation
 - Updated `blazepose` implementation
+- Strong typing for all string enums in `config` and `results`

src/body/blazepose.ts

@@ -5,7 +5,7 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import { constants } from '../tfjs/constants';
 import { log, join, now } from '../util/util';
-import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
+import type { BodyKeypoint, BodyResult, BodyLandmark, Box, Point, BodyAnnotation } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import * as coords from './blazeposecoords';
@@ -144,13 +144,13 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
     const adjScore = Math.trunc(100 * score * presence * poseScore) / 100;
     const positionRaw: Point = [points[depth * i + 0] / inputSize.landmarks[0], points[depth * i + 1] / inputSize.landmarks[1], points[depth * i + 2] + 0];
     const position: Point = [Math.trunc(outputSize[0] * positionRaw[0]), Math.trunc(outputSize[1] * positionRaw[1]), positionRaw[2] as number];
-    keypointsRelative.push({ part: coords.kpt[i], positionRaw, position, score: adjScore });
+    keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore });
   }
   if (poseScore < (config.body.minConfidence || 0)) return null;
   const keypoints: Array<BodyKeypoint> = rescaleKeypoints(keypointsRelative, outputSize); // keypoints were relative to input image which is padded
   const kpts = keypoints.map((k) => k.position);
   const boxes = box.calc(kpts, [outputSize[0], outputSize[1]]); // now find boxes based on rescaled keypoints
-  const annotations: Record<string, Point[][]> = {};
+  const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>;
   for (const [name, indexes] of Object.entries(coords.connected)) {
     const pt: Array<Point[]> = [];
     for (let i = 0; i < indexes.length - 1; i++) {

src/body/efficientpose.ts

@@ -8,14 +8,14 @@ import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './efficientposecoords';
 import { constants } from '../tfjs/constants';
-import type { BodyResult, Point } from '../result';
+import type { BodyResult, Point, BodyLandmark, BodyAnnotation } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
 let lastTime = 0;
-const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} };
+const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} as Record<BodyAnnotation, Point[][]> };
 
 // const keypoints: Array<BodyKeypoint> = [];
 // let box: Box = [0, 0, 0, 0];
@@ -88,7 +88,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
         if (partScore > (config.body?.minConfidence || 0)) {
           cache.keypoints.push({
             score: Math.round(100 * partScore) / 100,
-            part: coords.kpt[id],
+            part: coords.kpt[id] as BodyLandmark,
             positionRaw: [ // normalized to 0..1
               // @ts-ignore model is not undefined here
               x / model.inputs[0].shape[2], y / model.inputs[0].shape[1],

src/body/movenet.ts

@@ -9,7 +9,7 @@ import * as box from '../util/box';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './movenetcoords';
 import * as fix from './movenetfix';
-import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
+import type { BodyKeypoint, BodyResult, BodyLandmark, BodyAnnotation, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { fakeOps } from '../tfjs/backend';
@@ -52,7 +52,7 @@ async function parseSinglePose(res, config, image) {
       const positionRaw: Point = [kpt[id][1], kpt[id][0]];
       keypoints.push({
         score: Math.round(100 * score) / 100,
-        part: coords.kpt[id],
+        part: coords.kpt[id] as BodyLandmark,
         positionRaw,
         position: [ // normalized to input image size
           Math.round((image.shape[2] || 0) * positionRaw[0]),
@@ -92,7 +92,7 @@ async function parseMultiPose(res, config, image) {
         if (score > config.body.minConfidence) {
           const positionRaw: Point = [kpt[3 * i + 1], kpt[3 * i + 0]];
           keypoints.push({
-            part: coords.kpt[i],
+            part: coords.kpt[i] as BodyLandmark,
             score: Math.round(100 * score) / 100,
             positionRaw,
             position: [Math.round((image.shape[2] || 0) * positionRaw[0]), Math.round((image.shape[1] || 0) * positionRaw[1])],
@@ -103,7 +103,7 @@ async function parseMultiPose(res, config, image) {
       // movenet-multipose has built-in box details
       // const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
       // const box: Box = [Math.trunc(boxRaw[0] * (image.shape[2] || 0)), Math.trunc(boxRaw[1] * (image.shape[1] || 0)), Math.trunc(boxRaw[2] * (image.shape[2] || 0)), Math.trunc(boxRaw[3] * (image.shape[1] || 0))];
-      const annotations: Record<string, Point[][]> = {};
+      const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>;
       for (const [name, indexes] of Object.entries(coords.connected)) {
         const pt: Array<Point[]> = [];
         for (let i = 0; i < indexes.length - 1; i++) {

src/body/posenet.ts

@@ -6,7 +6,7 @@
 
 import { log, join } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
-import type { BodyResult, Box } from '../result';
+import type { BodyResult, BodyLandmark, Box } from '../result';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
@@ -14,7 +14,6 @@ import * as utils from './posenetutils';
 
 let model: GraphModel;
 const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd'/* offsets */, 'MobilenetV1/heatmap_2/BiasAdd'/* heatmapScores */, 'MobilenetV1/displacement_fwd_2/BiasAdd'/* displacementFwd */, 'MobilenetV1/displacement_bwd_2/BiasAdd'/* displacementBwd */];
-
 const localMaximumRadius = 1;
 const outputStride = 16;
 const squaredNmsRadius = 50 ** 2;
@@ -59,7 +58,7 @@ export function decodePose(root, scores, offsets, displacementsFwd, displacement
   const rootPoint = utils.getImageCoords(root.part, outputStride, offsets);
   keypoints[root.part.id] = {
     score: root.score,
-    part: utils.partNames[root.part.id],
+    part: utils.partNames[root.part.id] as BodyLandmark,
     position: rootPoint,
   };
   // Decode the part positions upwards in the tree, following the backward displacements.

src/body/posenetutils.ts

@@ -3,7 +3,7 @@
  * See `posenet.ts` for entry point
  */
 
-import type { BodyResult } from '../result';
+import type { Point, BodyResult, BodyAnnotation, BodyLandmark } from '../result';
 
 export const partNames = [
   'nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftShoulder',
@@ -71,17 +71,18 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
 export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<BodyResult> {
   const scaleY = height / inputResolutionHeight;
   const scaleX = width / inputResolutionWidth;
-  const scalePose = (pose, i) => ({
+  const scalePose = (pose, i): BodyResult => ({
     id: i,
     score: pose.score,
     boxRaw: [pose.box[0] / inputResolutionWidth, pose.box[1] / inputResolutionHeight, pose.box[2] / inputResolutionWidth, pose.box[3] / inputResolutionHeight],
     box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)],
     keypoints: pose.keypoints.map(({ score, part, position }) => ({
-      score,
+      score: score as number,
-      part,
+      part: part as BodyLandmark,
-      position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
+      position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)] as Point,
-      positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight],
+      positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight] as Point,
     })),
+    annotations: {} as Record<BodyAnnotation, Point[][]>,
   });
   const scaledPoses = poses.map((pose, i) => scalePose(pose, i));
   return scaledPoses;

src/config.ts

@@ -241,7 +241,6 @@ export interface Config {
    * default: `full`
   */
   warmup: '' | 'none' | 'face' | 'full' | 'body',
-  // warmup: string;
 
   /** Base model path (typically starting with file://, http:// or https://) for all models
    * - individual modelPath values are relative to this path

src/exports.ts

@@ -11,6 +11,7 @@ export type { Box, Point } from './result';
 export type { Models } from './models';
 export type { Env } from './util/env';
 export type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
+export type { Emotion, Finger, FingerCurl, FingerDirection, HandType, Gender, Race, FaceLandmark, BodyLandmark, BodyAnnotation, ObjectType } from './result';
 export { env } from './util/env';
 
 /** Events dispatched by `human.events`

src/face/face.ts

@@ -162,9 +162,9 @@ export const detectFace = async (instance: Human /* instance of human */, input:
 
     // calculate iris distance
     // iris: array[ center, left, top, right, bottom]
-    if (!instance.config.face.iris?.enabled && faces[i]?.annotations?.leftEyeIris && faces[i]?.annotations?.rightEyeIris) {
+    if (!instance.config.face.iris?.enabled) {
-      delete faces[i].annotations.leftEyeIris;
+      // if (faces[i]?.annotations?.leftEyeIris) delete faces[i].annotations.leftEyeIris;
-      delete faces[i].annotations.rightEyeIris;
+      // if (faces[i]?.annotations?.rightEyeIris) delete faces[i].annotations.rightEyeIris;
     }
     const irisSize = (faces[i].annotations && faces[i].annotations.leftEyeIris && faces[i].annotations.leftEyeIris[0] && faces[i].annotations.rightEyeIris && faces[i].annotations.rightEyeIris[0]
       && (faces[i].annotations.leftEyeIris.length > 0) && (faces[i].annotations.rightEyeIris.length > 0)

src/face/facemesh.ts

@@ -16,7 +16,7 @@ import * as iris from './iris';
 import { histogramEqualization } from '../image/enhance';
 import { env } from '../util/env';
 import type { GraphModel, Tensor } from '../tfjs/types';
-import type { FaceResult, Point } from '../result';
+import type { FaceResult, FaceLandmark, Point } from '../result';
 import type { Config } from '../config';
 
 type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
@@ -62,7 +62,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
       score: 0,
       boxScore: 0,
       faceScore: 0,
-      annotations: {},
+      annotations: {} as Record<FaceLandmark, Point[]>,
     };
 
     // optional rotation correction based on detector data only if mesh is disabled otherwise perform it later when we have more accurate mesh data. if no rotation correction this function performs crop

src/gear/emotion.ts

@@ -4,6 +4,7 @@
  * [**Oarriaga**](https://github.com/oarriaga/face_classification)
  */
 
+import type { Emotion } from '../result';
 import { log, join, now } from '../util/util';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
@@ -13,7 +14,7 @@ import { constants } from '../tfjs/constants';
 
 const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
 let model: GraphModel | null;
-const last: Array<Array<{ score: number, emotion: string }>> = [];
+const last: Array<Array<{ score: number, emotion: Emotion }>> = [];
 let lastCount = 0;
 let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
@@ -28,7 +29,7 @@ export async function load(config: Config): Promise<GraphModel> {
   return model;
 }
 
-export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<Array<{ score: number, emotion: string }>> {
+export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<Array<{ score: number, emotion: Emotion }>> {
   if (!model) return [];
   const skipFrame = skipped < (config.face.emotion?.skipFrames || 0);
   const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime);
@@ -38,7 +39,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count:
   }
   skipped = 0;
   return new Promise(async (resolve) => {
-    const obj: Array<{ score: number, emotion: string }> = [];
+    const obj: Array<{ score: number, emotion: Emotion }> = [];
     if (config.face.emotion?.enabled) {
       const t: Record<string, Tensor> = {};
       const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0;
@@ -59,7 +60,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count:
       lastTime = now();
       const data = await t.emotion.data();
       for (let i = 0; i < data.length; i++) {
-        if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
+        if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] as Emotion });
       }
       obj.sort((a, b) => b.score - a.score);
       Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));

src/gear/gear.ts

@@ -6,11 +6,12 @@
 
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
+import type { Gender, Race } from '../result';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import { env } from '../util/env';
 
-type GearType = { age: number, gender: string, genderScore: number, race: Array<{ score: number, race: string }> }
+type GearType = { age: number, gender: Gender, genderScore: number, race: Array<{ score: number, race: Race }> }
 let model: GraphModel | null;
 const last: Array<GearType> = [];
 const raceNames = ['white', 'black', 'asian', 'indian', 'other'];
@@ -53,7 +54,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis
     obj.genderScore = Math.round(100 * (gender[0] > gender[1] ? gender[0] : gender[1])) / 100;
     const race = await t.race.data();
     for (let i = 0; i < race.length; i++) {
-      if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] });
+      if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] as Race });
     }
     obj.race.sort((a, b) => b.score - a.score);
     // {0: 'Below20', 1: '21-25', 2: '26-30', 3: '31-40',4: '41-50', 5: '51-60', 6: 'Above60'}

src/gear/ssrnet-gender.ts

@@ -7,12 +7,13 @@
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { constants } from '../tfjs/constants';
+import type { Gender } from '../result';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import { env } from '../util/env';
 
 let model: GraphModel | null;
-const last: Array<{ gender: string, genderScore: number }> = [];
+const last: Array<{ gender: Gender, genderScore: number }> = [];
 let lastCount = 0;
 let lastTime = 0;
 let skipped = Number.MAX_SAFE_INTEGER;
@@ -32,7 +33,7 @@ export async function load(config: Config | any) {
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: string, genderScore: number }> {
+export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: Gender, genderScore: number }> {
   if (!model) return { gender: 'unknown', genderScore: 0 };
   const skipFrame = skipped < (config.face['ssrnet']?.skipFrames || 0);
   const skipTime = (config.face['ssrnet']?.skipTime || 0) > (now() - lastTime);
@@ -54,7 +55,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis
       const normalize = tf.mul(tf.sub(grayscale, constants.tf05), 2); // range grayscale:-1..1
       return normalize;
     });
-    const obj = { gender: '', genderScore: 0 };
+    const obj: { gender: Gender, genderScore: number } = { gender: 'unknown', genderScore: 0 };
     if (config.face['ssrnet'].enabled) t.gender = model.execute(t.enhance) as Tensor;
     const data = await t.gender.data();
     obj.gender = data[0] > data[1] ? 'female' : 'male'; // returns two values 0..1, bigger one is prediction

src/hand/handtrack.ts

@@ -9,7 +9,7 @@
 import { log, join, now } from '../util/util';
 import * as box from '../util/box';
 import * as tf from '../../dist/tfjs.esm.js';
-import type { HandResult, Box, Point } from '../result';
+import type { HandResult, HandType, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
@@ -39,7 +39,7 @@ type HandDetectResult = {
   box: Box,
   boxRaw: Box,
   boxCrop: Box,
-  label: string,
+  label: HandType,
 }
 
 const cache: {
@@ -129,7 +129,7 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
     const boxCrop: Box = box.crop(boxRaw); // crop box is based on raw box
     const boxFull: Box = [Math.trunc(boxData[0] * outputSize[0]), Math.trunc(boxData[1] * outputSize[1]), Math.trunc(boxData[2] * outputSize[0]), Math.trunc(boxData[3] * outputSize[1])];
     const score = scores[nmsIndex];
-    const label = classes[classNum[nmsIndex]];
+    const label = classes[classNum[nmsIndex]] as HandType;
     const hand: HandDetectResult = { id: id++, score, box: boxFull, boxRaw, boxCrop, label };
     hands.push(hand);
   }

src/object/centernet.ts

@@ -7,7 +7,7 @@
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { labels } from './labels';
-import type { ObjectResult, Box } from '../result';
+import type { ObjectResult, ObjectType, Box } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
@@ -49,7 +49,7 @@ async function process(res: Tensor | null, outputShape, config: Config) {
   for (const id of Array.from(nms)) {
     const score = Math.trunc(100 * detections[0][id][4]) / 100;
     const classVal = detections[0][id][5];
-    const label = labels[classVal].label;
+    const label = labels[classVal].label as ObjectType;
     const [x, y] = [
       detections[0][id][0] / inputSize,
       detections[0][id][1] / inputSize,

src/object/nanodet.ts

@@ -8,7 +8,7 @@ import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { constants } from '../tfjs/constants';
 import { labels } from './labels';
-import type { ObjectResult, Box } from '../result';
+import type { ObjectResult, ObjectType, Box } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';
 import { env } from '../util/env';
@@ -72,7 +72,7 @@ async function process(res, inputSize, outputShape, config) {
               // strideSize,
               score: Math.round(100 * score) / 100,
               class: j + 1,
-              label: labels[j].label,
+              label: labels[j].label as ObjectType,
               // center: [Math.trunc(outputShape[0] * cx), Math.trunc(outputShape[1] * cy)],
               // centerRaw: [cx, cy],
               box: box.map((a) => Math.trunc(a)) as Box,

src/result.ts

@@ -11,6 +11,15 @@ export type Box = [number, number, number, number];
 /** generic point as [x, y, z?] */
 export type Point = [number, number, number?];
 
+export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral';
+export type Gender = 'male' | 'female' | 'unknown';
+export type Race = 'white' | 'black' | 'asian' | 'indian' | 'other';
+export type FaceLandmark = 'leftEye' | 'rightEye' | 'nose' | 'mouth' | 'leftEar' | 'rightEar' | 'symmetryLine' | 'silhouette'
+ | 'lipsUpperOuter' | 'lipsLowerOuter' | 'lipsUpperInner' | 'lipsLowerInner'
+ | 'rightEyeUpper0' | 'rightEyeLower0' | 'rightEyeUpper1' | 'rightEyeLower1' | 'rightEyeUpper2' | 'rightEyeLower2' | 'rightEyeLower3' | 'rightEyebrowUpper' | 'rightEyebrowLower' | 'rightEyeIris'
+ | 'leftEyeUpper0' | 'leftEyeLower0' | 'leftEyeUpper1' | 'leftEyeLower1' | 'leftEyeUpper2' | 'leftEyeLower2' | 'leftEyeLower3' | 'leftEyebrowUpper' | 'leftEyebrowLower' | 'leftEyeIris'
+ | 'midwayBetweenEyes' | 'noseTip' | 'noseBottom' | 'noseRightCorner' | 'noseLeftCorner' | 'rightCheek' | 'leftCheek';
+
 /** Face results
  * - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
  * - Some values may be null if specific model is not enabled
@@ -33,17 +42,17 @@ export interface FaceResult {
   /** detected face mesh normalized to 0..1 */
   meshRaw: Array<Point>
   /** mesh keypoints combined into annotated results */
-  annotations: Record<string, Point[]>,
+  annotations: Record<FaceLandmark, Point[]>,
   /** detected age */
   age?: number,
   /** detected gender */
-  gender?: string,
+  gender?: Gender,
   /** gender detection score */
   genderScore?: number,
   /** detected emotions */
-  emotion?: Array<{ score: number, emotion: string }>,
+  emotion?: Array<{ score: number, emotion: Emotion }>,
   /** detected race */
-  race?: Array<{ score: number, race: string }>,
+  race?: Array<{ score: number, race: Race }>,
   /** face descriptor */
   embedding?: Array<number>,
   /** face iris distance from camera */
@@ -62,10 +71,21 @@ export interface FaceResult {
   tensor?: Tensor,
 }
 
+export type BodyLandmarkPoseNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
+export type BodyLandmarkMoveNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
+export type BodyLandmarkEfficientNet = 'head' | 'neck' | 'rightShoulder' | 'rightElbow' | 'rightWrist' | 'chest' | 'leftShoulder' | 'leftElbow' | 'leftWrist' | 'bodyCenter' | 'rightHip' | 'rightKnee' | 'rightAnkle' | 'leftHip' | 'leftKnee' | 'leftAnkle';
+export type BodyLandmarkBlazePose = 'nose' | 'leftEyeInside' | 'leftEye' | 'leftEyeOutside' | 'rightEyeInside' | 'rightEye' | 'rightEyeOutside' | 'leftEar' | 'rightEar' | 'leftMouth' | 'rightMouth' | 'leftShoulder' | 'rightShoulder'
+  | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftPinky' | 'rightPinky' | 'leftIndex' | 'rightIndex' | 'leftThumb' | 'rightThumb' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle'
+  | 'leftHeel' | 'rightHeel' | 'leftFoot' | 'rightFoot' | 'bodyCenter' | 'bodyTop' | 'leftPalm' | 'leftHand' | 'rightPalm' | 'rightHand';
+export type BodyLandmark = BodyLandmarkPoseNet | BodyLandmarkMoveNet | BodyLandmarkEfficientNet | BodyLandmarkBlazePose;
+export type BodyAnnotationBlazePose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'leftEye' | 'rightEye' | 'mouth';
+export type BodyAnnotationEfficientPose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'head';
+export type BodyAnnotation = BodyAnnotationBlazePose | BodyAnnotationEfficientPose;
+
 /** Body Result keypoints */
 export interface BodyKeypoint {
   /** body part name */
-  part: string,
+  part: BodyLandmark,
   /** body part position */
   position: Point,
   /** body part position normalized to 0..1 */
@@ -87,9 +107,14 @@ export interface BodyResult {
   /** detected body keypoints */
   keypoints: Array<BodyKeypoint>
   /** detected body keypoints combined into annotated parts */
-  annotations: Record<string, Array<Point[]>>,
+  annotations: Record<BodyAnnotation, Point[][]>,
 }
 
+export type HandType = 'hand' | 'fist' | 'pinch' | 'point' | 'face' | 'tip' | 'pinchtip';
+export type Finger = 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm';
+export type FingerCurl = 'none' | 'half' | 'full';
+export type FingerDirection = 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft';
+
 /** Hand results */
 export interface HandResult {
   /** hand id */
@@ -107,19 +132,20 @@ export interface HandResult {
   /** detected hand keypoints */
   keypoints: Array<Point>,
   /** detected hand class */
-  label: string,
+  label: HandType,
   /** detected hand keypoints combined into annotated parts */
-  annotations: Record<
+  annotations: Record<Finger, Array<Point>>,
-    'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
-    Array<Point>
-  >,
   /** detected hand parts annotated with part gestures */
-  landmarks: Record<
+  landmarks: Record<Finger, { curl: FingerCurl, direction: FingerDirection }>,
-    'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
-    { curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
-  >,
 }
 
+export type ObjectType = 'person' | 'bicycle' | 'car' | 'motorcycle' | 'airplane' | 'bus' | 'train' | 'truck' | 'boat' | 'traffic light' | 'fire hydrant' | 'stop sign' | 'parking meter'
+  | 'bench' | 'bird' | 'cat' | 'dog' | 'horse' | 'sheep' | 'cow' | 'elephant' | 'bear' | 'zebra' | 'giraffe' | 'backpack' | 'umbrella' | 'handbag' | 'tie' | 'suitcase' | 'frisbee'
+  | 'skis' | 'snowboard' | 'sports ball' | 'kite' | 'baseball bat' | 'baseball glove' | 'skateboard' | 'surfboard' | 'tennis racket' | 'bottle' | 'wine glass' | 'cup' | 'fork'
+  | 'knife' | 'spoon' | 'bowl' | 'banana' | 'apple' | 'sandwich' | 'orange' | 'broccoli' | 'carrot' | 'hot dog' | 'pizza' | 'donut' | 'cake' | 'chair' | 'couch' | 'potted plant'
+  | 'bed' | 'dining table' | 'toilet' | 'tv' | 'laptop' | 'mouse' | 'remote' | 'keyboard' | 'cell phone' | 'microwave' | 'oven' | 'toaster' | 'sink' | 'refrigerator' | 'book'
+  | 'clock' | 'vase' | 'scissors' | 'teddy bear' | 'hair drier' | 'toothbrush';
+
 /** Object results */
 export interface ObjectResult {
   /** object id */
@@ -129,7 +155,7 @@ export interface ObjectResult {
   /** detected object class id */
   class: number,
   /** detected object class name */
-  label: string,
+  label: ObjectType,
   /** detected object box */
   box: Box,
   /** detected object box normalized to 0..1 */

src/util/interpolate.ts

@@ -2,7 +2,7 @@
  * Results interpolation for smoothening of video detection results inbetween detected frames
  */
 
-import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point } from '../result';
+import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point, BodyLandmark, BodyAnnotation } from '../result';
 import type { Config } from '../config';
 
 import * as moveNetCoords from '../body/movenetcoords';
@@ -46,7 +46,7 @@ export function calc(newResult: Result, config: Config): Result {
       const keypoints = (newResult.body[i].keypoints // update keypoints
         .map((newKpt, j) => ({
           score: newKpt.score,
-          part: newKpt.part,
+          part: newKpt.part as BodyLandmark,
           position: [
             bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[0] || 0) + (newKpt.position[0] || 0)) / bufferedFactor : newKpt.position[0],
             bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[1] || 0) + (newKpt.position[1] || 0)) / bufferedFactor : newKpt.position[1],
@@ -57,9 +57,9 @@ export function calc(newResult: Result, config: Config): Result {
             bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1],
             bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2],
           ],
-        }))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
+        }))) as Array<{ score: number, part: BodyLandmark, position: [number, number, number?], positionRaw: [number, number, number?] }>;
 
-      const annotations: Record<string, Point[][]> = {}; // recreate annotations
+      const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>; // recreate annotations
       let coords = { connected: {} };
       if (config.body?.modelPath?.includes('efficientpose')) coords = efficientPoseCoords;
       else if (config.body?.modelPath?.includes('blazepose')) coords = blazePoseCoords;