diff --git a/CHANGELOG.md b/CHANGELOG.md index 8431b0d0..d1f29d11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ ## Changelog -### **HEAD -> main** 2021/12/13 mandic00@live.com +### **HEAD -> main** 2021/12/14 mandic00@live.com +- rebuild - fix node detection in electron environment ### **2.5.5** 2021/12/01 mandic00@live.com diff --git a/TODO.md b/TODO.md index 65f21004..2a7a9d6a 100644 --- a/TODO.md +++ b/TODO.md @@ -10,7 +10,6 @@ - Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE - TFLite models: - Body segmentation: `robust-video-matting` -- TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch

@@ -57,3 +56,4 @@ Other: - Fix face detect box scale and rotation - Fix body interpolation - Updated `blazepose` implementation +- Strong typing for all string enums in `config` and `results` diff --git a/src/body/blazepose.ts b/src/body/blazepose.ts index b21f8649..8208a186 100644 --- a/src/body/blazepose.ts +++ b/src/body/blazepose.ts @@ -5,7 +5,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import { constants } from '../tfjs/constants'; import { log, join, now } from '../util/util'; -import type { BodyKeypoint, BodyResult, Box, Point } from '../result'; +import type { BodyKeypoint, BodyResult, BodyLandmark, Box, Point, BodyAnnotation } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import * as coords from './blazeposecoords'; @@ -144,13 +144,13 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe const adjScore = Math.trunc(100 * score * presence * poseScore) / 100; const positionRaw: Point = [points[depth * i + 0] / inputSize.landmarks[0], points[depth * i + 1] / inputSize.landmarks[1], points[depth * i + 2] + 0]; const position: Point = [Math.trunc(outputSize[0] * positionRaw[0]), Math.trunc(outputSize[1] * positionRaw[1]), positionRaw[2] as number]; - keypointsRelative.push({ part: coords.kpt[i], positionRaw, position, score: adjScore }); + keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore }); } if (poseScore < (config.body.minConfidence || 0)) return null; const keypoints: Array = rescaleKeypoints(keypointsRelative, outputSize); // keypoints were relative to input image which is padded const kpts = keypoints.map((k) => k.position); const boxes = box.calc(kpts, [outputSize[0], outputSize[1]]); // now find boxes based on rescaled keypoints - const annotations: Record = {}; + const annotations: Record = {} as Record; for (const [name, indexes] of Object.entries(coords.connected)) { const pt: Array = []; for (let i = 0; i < indexes.length - 1; i++) { diff --git a/src/body/efficientpose.ts b/src/body/efficientpose.ts index b70e0d0f..2ddd946e 100644 --- a/src/body/efficientpose.ts +++ b/src/body/efficientpose.ts @@ -8,14 +8,14 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import * as coords from './efficientposecoords'; import { constants } from '../tfjs/constants'; -import type { BodyResult, Point } from '../result'; +import type { BodyResult, Point, BodyLandmark, BodyAnnotation } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; let model: GraphModel | null; let lastTime = 0; -const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} }; +const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} as Record }; // const keypoints: Array = []; // let box: Box = [0, 0, 0, 0]; @@ -88,7 +88,7 @@ export async function predict(image: Tensor, config: Config): Promise (config.body?.minConfidence || 0)) { cache.keypoints.push({ score: Math.round(100 * partScore) / 100, - part: coords.kpt[id], + part: coords.kpt[id] as BodyLandmark, positionRaw: [ // normalized to 0..1 // @ts-ignore model is not undefined here x / model.inputs[0].shape[2], y / model.inputs[0].shape[1], diff --git a/src/body/movenet.ts b/src/body/movenet.ts index cdb92663..0d061990 100644 --- a/src/body/movenet.ts +++ b/src/body/movenet.ts @@ -9,7 +9,7 @@ import * as box from '../util/box'; import * as tf from '../../dist/tfjs.esm.js'; import * as coords from './movenetcoords'; import * as fix from './movenetfix'; -import type { BodyKeypoint, BodyResult, Box, Point } from '../result'; +import type { BodyKeypoint, BodyResult, BodyLandmark, BodyAnnotation, Box, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { fakeOps } from '../tfjs/backend'; @@ -52,7 +52,7 @@ async function parseSinglePose(res, config, image) { const positionRaw: Point = [kpt[id][1], kpt[id][0]]; keypoints.push({ score: Math.round(100 * score) / 100, - part: coords.kpt[id], + part: coords.kpt[id] as BodyLandmark, positionRaw, position: [ // normalized to input image size Math.round((image.shape[2] || 0) * positionRaw[0]), @@ -92,7 +92,7 @@ async function parseMultiPose(res, config, image) { if (score > config.body.minConfidence) { const positionRaw: Point = [kpt[3 * i + 1], kpt[3 * i + 0]]; keypoints.push({ - part: coords.kpt[i], + part: coords.kpt[i] as BodyLandmark, score: Math.round(100 * score) / 100, positionRaw, position: [Math.round((image.shape[2] || 0) * positionRaw[0]), Math.round((image.shape[1] || 0) * positionRaw[1])], @@ -103,7 +103,7 @@ async function parseMultiPose(res, config, image) { // movenet-multipose has built-in box details // const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]]; // const box: Box = [Math.trunc(boxRaw[0] * (image.shape[2] || 0)), Math.trunc(boxRaw[1] * (image.shape[1] || 0)), Math.trunc(boxRaw[2] * (image.shape[2] || 0)), Math.trunc(boxRaw[3] * (image.shape[1] || 0))]; - const annotations: Record = {}; + const annotations: Record = {} as Record; for (const [name, indexes] of Object.entries(coords.connected)) { const pt: Array = []; for (let i = 0; i < indexes.length - 1; i++) { diff --git a/src/body/posenet.ts b/src/body/posenet.ts index e7bd4703..724aa69c 100644 --- a/src/body/posenet.ts +++ b/src/body/posenet.ts @@ -6,7 +6,7 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; -import type { BodyResult, Box } from '../result'; +import type { BodyResult, BodyLandmark, Box } from '../result'; import type { Tensor, GraphModel } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; @@ -14,7 +14,6 @@ import * as utils from './posenetutils'; let model: GraphModel; const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd'/* offsets */, 'MobilenetV1/heatmap_2/BiasAdd'/* heatmapScores */, 'MobilenetV1/displacement_fwd_2/BiasAdd'/* displacementFwd */, 'MobilenetV1/displacement_bwd_2/BiasAdd'/* displacementBwd */]; - const localMaximumRadius = 1; const outputStride = 16; const squaredNmsRadius = 50 ** 2; @@ -59,7 +58,7 @@ export function decodePose(root, scores, offsets, displacementsFwd, displacement const rootPoint = utils.getImageCoords(root.part, outputStride, offsets); keypoints[root.part.id] = { score: root.score, - part: utils.partNames[root.part.id], + part: utils.partNames[root.part.id] as BodyLandmark, position: rootPoint, }; // Decode the part positions upwards in the tree, following the backward displacements. diff --git a/src/body/posenetutils.ts b/src/body/posenetutils.ts index 1d963692..3b71410a 100644 --- a/src/body/posenetutils.ts +++ b/src/body/posenetutils.ts @@ -3,7 +3,7 @@ * See `posenet.ts` for entry point */ -import type { BodyResult } from '../result'; +import type { Point, BodyResult, BodyAnnotation, BodyLandmark } from '../result'; export const partNames = [ 'nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftShoulder', @@ -71,17 +71,18 @@ export function getBoundingBox(keypoints): [number, number, number, number] { export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array { const scaleY = height / inputResolutionHeight; const scaleX = width / inputResolutionWidth; - const scalePose = (pose, i) => ({ + const scalePose = (pose, i): BodyResult => ({ id: i, score: pose.score, boxRaw: [pose.box[0] / inputResolutionWidth, pose.box[1] / inputResolutionHeight, pose.box[2] / inputResolutionWidth, pose.box[3] / inputResolutionHeight], box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)], keypoints: pose.keypoints.map(({ score, part, position }) => ({ - score, - part, - position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)], - positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight], + score: score as number, + part: part as BodyLandmark, + position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)] as Point, + positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight] as Point, })), + annotations: {} as Record, }); const scaledPoses = poses.map((pose, i) => scalePose(pose, i)); return scaledPoses; diff --git a/src/config.ts b/src/config.ts index 2ffa6b4c..edc3a658 100644 --- a/src/config.ts +++ b/src/config.ts @@ -241,7 +241,6 @@ export interface Config { * default: `full` */ warmup: '' | 'none' | 'face' | 'full' | 'body', - // warmup: string; /** Base model path (typically starting with file://, http:// or https://) for all models * - individual modelPath values are relative to this path diff --git a/src/exports.ts b/src/exports.ts index 76cd55c2..973f4972 100644 --- a/src/exports.ts +++ b/src/exports.ts @@ -11,6 +11,7 @@ export type { Box, Point } from './result'; export type { Models } from './models'; export type { Env } from './util/env'; export type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; +export type { Emotion, Finger, FingerCurl, FingerDirection, HandType, Gender, Race, FaceLandmark, BodyLandmark, BodyAnnotation, ObjectType } from './result'; export { env } from './util/env'; /** Events dispatched by `human.events` diff --git a/src/face/face.ts b/src/face/face.ts index 078c289d..e4dc69e1 100644 --- a/src/face/face.ts +++ b/src/face/face.ts @@ -162,9 +162,9 @@ export const detectFace = async (instance: Human /* instance of human */, input: // calculate iris distance // iris: array[ center, left, top, right, bottom] - if (!instance.config.face.iris?.enabled && faces[i]?.annotations?.leftEyeIris && faces[i]?.annotations?.rightEyeIris) { - delete faces[i].annotations.leftEyeIris; - delete faces[i].annotations.rightEyeIris; + if (!instance.config.face.iris?.enabled) { + // if (faces[i]?.annotations?.leftEyeIris) delete faces[i].annotations.leftEyeIris; + // if (faces[i]?.annotations?.rightEyeIris) delete faces[i].annotations.rightEyeIris; } const irisSize = (faces[i].annotations && faces[i].annotations.leftEyeIris && faces[i].annotations.leftEyeIris[0] && faces[i].annotations.rightEyeIris && faces[i].annotations.rightEyeIris[0] && (faces[i].annotations.leftEyeIris.length > 0) && (faces[i].annotations.rightEyeIris.length > 0) diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index ae86ee58..405b4ada 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -16,7 +16,7 @@ import * as iris from './iris'; import { histogramEqualization } from '../image/enhance'; import { env } from '../util/env'; import type { GraphModel, Tensor } from '../tfjs/types'; -import type { FaceResult, Point } from '../result'; +import type { FaceResult, FaceLandmark, Point } from '../result'; import type { Config } from '../config'; type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array, confidence: number }; @@ -62,7 +62,7 @@ export async function predict(input: Tensor, config: Config): Promise, }; // optional rotation correction based on detector data only if mesh is disabled otherwise perform it later when we have more accurate mesh data. if no rotation correction this function performs crop diff --git a/src/gear/emotion.ts b/src/gear/emotion.ts index ab69f835..4992b545 100644 --- a/src/gear/emotion.ts +++ b/src/gear/emotion.ts @@ -4,6 +4,7 @@ * [**Oarriaga**](https://github.com/oarriaga/face_classification) */ +import type { Emotion } from '../result'; import { log, join, now } from '../util/util'; import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; @@ -13,7 +14,7 @@ import { constants } from '../tfjs/constants'; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; let model: GraphModel | null; -const last: Array> = []; +const last: Array> = []; let lastCount = 0; let lastTime = 0; let skipped = Number.MAX_SAFE_INTEGER; @@ -28,7 +29,7 @@ export async function load(config: Config): Promise { return model; } -export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise> { +export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise> { if (!model) return []; const skipFrame = skipped < (config.face.emotion?.skipFrames || 0); const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime); @@ -38,7 +39,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count: } skipped = 0; return new Promise(async (resolve) => { - const obj: Array<{ score: number, emotion: string }> = []; + const obj: Array<{ score: number, emotion: Emotion }> = []; if (config.face.emotion?.enabled) { const t: Record = {}; const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0; @@ -59,7 +60,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count: lastTime = now(); const data = await t.emotion.data(); for (let i = 0; i < data.length; i++) { - if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); + if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] as Emotion }); } obj.sort((a, b) => b.score - a.score); Object.keys(t).forEach((tensor) => tf.dispose(t[tensor])); diff --git a/src/gear/gear.ts b/src/gear/gear.ts index 040a6e89..1d7e5199 100644 --- a/src/gear/gear.ts +++ b/src/gear/gear.ts @@ -6,11 +6,12 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; +import type { Gender, Race } from '../result'; import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; import { env } from '../util/env'; -type GearType = { age: number, gender: string, genderScore: number, race: Array<{ score: number, race: string }> } +type GearType = { age: number, gender: Gender, genderScore: number, race: Array<{ score: number, race: Race }> } let model: GraphModel | null; const last: Array = []; const raceNames = ['white', 'black', 'asian', 'indian', 'other']; @@ -53,7 +54,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis obj.genderScore = Math.round(100 * (gender[0] > gender[1] ? gender[0] : gender[1])) / 100; const race = await t.race.data(); for (let i = 0; i < race.length; i++) { - if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] }); + if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] as Race }); } obj.race.sort((a, b) => b.score - a.score); // {0: 'Below20', 1: '21-25', 2: '26-30', 3: '31-40',4: '41-50', 5: '51-60', 6: 'Above60'} diff --git a/src/gear/ssrnet-gender.ts b/src/gear/ssrnet-gender.ts index 1fa10c0b..7cd38fb8 100644 --- a/src/gear/ssrnet-gender.ts +++ b/src/gear/ssrnet-gender.ts @@ -7,12 +7,13 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import { constants } from '../tfjs/constants'; +import type { Gender } from '../result'; import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; import { env } from '../util/env'; let model: GraphModel | null; -const last: Array<{ gender: string, genderScore: number }> = []; +const last: Array<{ gender: Gender, genderScore: number }> = []; let lastCount = 0; let lastTime = 0; let skipped = Number.MAX_SAFE_INTEGER; @@ -32,7 +33,7 @@ export async function load(config: Config | any) { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: string, genderScore: number }> { +export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: Gender, genderScore: number }> { if (!model) return { gender: 'unknown', genderScore: 0 }; const skipFrame = skipped < (config.face['ssrnet']?.skipFrames || 0); const skipTime = (config.face['ssrnet']?.skipTime || 0) > (now() - lastTime); @@ -54,7 +55,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis const normalize = tf.mul(tf.sub(grayscale, constants.tf05), 2); // range grayscale:-1..1 return normalize; }); - const obj = { gender: '', genderScore: 0 }; + const obj: { gender: Gender, genderScore: number } = { gender: 'unknown', genderScore: 0 }; if (config.face['ssrnet'].enabled) t.gender = model.execute(t.enhance) as Tensor; const data = await t.gender.data(); obj.gender = data[0] > data[1] ? 'female' : 'male'; // returns two values 0..1, bigger one is prediction diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index 6af74e69..0b7d01ea 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -9,7 +9,7 @@ import { log, join, now } from '../util/util'; import * as box from '../util/box'; import * as tf from '../../dist/tfjs.esm.js'; -import type { HandResult, Box, Point } from '../result'; +import type { HandResult, HandType, Box, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; @@ -39,7 +39,7 @@ type HandDetectResult = { box: Box, boxRaw: Box, boxCrop: Box, - label: string, + label: HandType, } const cache: { @@ -129,7 +129,7 @@ async function detectHands(input: Tensor, config: Config): Promise Math.trunc(a)) as Box, diff --git a/src/result.ts b/src/result.ts index b4911e60..71682cca 100644 --- a/src/result.ts +++ b/src/result.ts @@ -11,6 +11,15 @@ export type Box = [number, number, number, number]; /** generic point as [x, y, z?] */ export type Point = [number, number, number?]; +export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral'; +export type Gender = 'male' | 'female' | 'unknown'; +export type Race = 'white' | 'black' | 'asian' | 'indian' | 'other'; +export type FaceLandmark = 'leftEye' | 'rightEye' | 'nose' | 'mouth' | 'leftEar' | 'rightEar' | 'symmetryLine' | 'silhouette' + | 'lipsUpperOuter' | 'lipsLowerOuter' | 'lipsUpperInner' | 'lipsLowerInner' + | 'rightEyeUpper0' | 'rightEyeLower0' | 'rightEyeUpper1' | 'rightEyeLower1' | 'rightEyeUpper2' | 'rightEyeLower2' | 'rightEyeLower3' | 'rightEyebrowUpper' | 'rightEyebrowLower' | 'rightEyeIris' + | 'leftEyeUpper0' | 'leftEyeLower0' | 'leftEyeUpper1' | 'leftEyeLower1' | 'leftEyeUpper2' | 'leftEyeLower2' | 'leftEyeLower3' | 'leftEyebrowUpper' | 'leftEyebrowLower' | 'leftEyeIris' + | 'midwayBetweenEyes' | 'noseTip' | 'noseBottom' | 'noseRightCorner' | 'noseLeftCorner' | 'rightCheek' | 'leftCheek'; + /** Face results * - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models * - Some values may be null if specific model is not enabled @@ -33,17 +42,17 @@ export interface FaceResult { /** detected face mesh normalized to 0..1 */ meshRaw: Array /** mesh keypoints combined into annotated results */ - annotations: Record, + annotations: Record, /** detected age */ age?: number, /** detected gender */ - gender?: string, + gender?: Gender, /** gender detection score */ genderScore?: number, /** detected emotions */ - emotion?: Array<{ score: number, emotion: string }>, + emotion?: Array<{ score: number, emotion: Emotion }>, /** detected race */ - race?: Array<{ score: number, race: string }>, + race?: Array<{ score: number, race: Race }>, /** face descriptor */ embedding?: Array, /** face iris distance from camera */ @@ -62,10 +71,21 @@ export interface FaceResult { tensor?: Tensor, } +export type BodyLandmarkPoseNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle'; +export type BodyLandmarkMoveNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle'; +export type BodyLandmarkEfficientNet = 'head' | 'neck' | 'rightShoulder' | 'rightElbow' | 'rightWrist' | 'chest' | 'leftShoulder' | 'leftElbow' | 'leftWrist' | 'bodyCenter' | 'rightHip' | 'rightKnee' | 'rightAnkle' | 'leftHip' | 'leftKnee' | 'leftAnkle'; +export type BodyLandmarkBlazePose = 'nose' | 'leftEyeInside' | 'leftEye' | 'leftEyeOutside' | 'rightEyeInside' | 'rightEye' | 'rightEyeOutside' | 'leftEar' | 'rightEar' | 'leftMouth' | 'rightMouth' | 'leftShoulder' | 'rightShoulder' + | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftPinky' | 'rightPinky' | 'leftIndex' | 'rightIndex' | 'leftThumb' | 'rightThumb' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle' + | 'leftHeel' | 'rightHeel' | 'leftFoot' | 'rightFoot' | 'bodyCenter' | 'bodyTop' | 'leftPalm' | 'leftHand' | 'rightPalm' | 'rightHand'; +export type BodyLandmark = BodyLandmarkPoseNet | BodyLandmarkMoveNet | BodyLandmarkEfficientNet | BodyLandmarkBlazePose; +export type BodyAnnotationBlazePose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'leftEye' | 'rightEye' | 'mouth'; +export type BodyAnnotationEfficientPose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'head'; +export type BodyAnnotation = BodyAnnotationBlazePose | BodyAnnotationEfficientPose; + /** Body Result keypoints */ export interface BodyKeypoint { /** body part name */ - part: string, + part: BodyLandmark, /** body part position */ position: Point, /** body part position normalized to 0..1 */ @@ -87,9 +107,14 @@ export interface BodyResult { /** detected body keypoints */ keypoints: Array /** detected body keypoints combined into annotated parts */ - annotations: Record>, + annotations: Record, } +export type HandType = 'hand' | 'fist' | 'pinch' | 'point' | 'face' | 'tip' | 'pinchtip'; +export type Finger = 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm'; +export type FingerCurl = 'none' | 'half' | 'full'; +export type FingerDirection = 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft'; + /** Hand results */ export interface HandResult { /** hand id */ @@ -107,19 +132,20 @@ export interface HandResult { /** detected hand keypoints */ keypoints: Array, /** detected hand class */ - label: string, + label: HandType, /** detected hand keypoints combined into annotated parts */ - annotations: Record< - 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm', - Array - >, + annotations: Record>, /** detected hand parts annotated with part gestures */ - landmarks: Record< - 'index' | 'middle' | 'pinky' | 'ring' | 'thumb', - { curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' } - >, + landmarks: Record, } +export type ObjectType = 'person' | 'bicycle' | 'car' | 'motorcycle' | 'airplane' | 'bus' | 'train' | 'truck' | 'boat' | 'traffic light' | 'fire hydrant' | 'stop sign' | 'parking meter' + | 'bench' | 'bird' | 'cat' | 'dog' | 'horse' | 'sheep' | 'cow' | 'elephant' | 'bear' | 'zebra' | 'giraffe' | 'backpack' | 'umbrella' | 'handbag' | 'tie' | 'suitcase' | 'frisbee' + | 'skis' | 'snowboard' | 'sports ball' | 'kite' | 'baseball bat' | 'baseball glove' | 'skateboard' | 'surfboard' | 'tennis racket' | 'bottle' | 'wine glass' | 'cup' | 'fork' + | 'knife' | 'spoon' | 'bowl' | 'banana' | 'apple' | 'sandwich' | 'orange' | 'broccoli' | 'carrot' | 'hot dog' | 'pizza' | 'donut' | 'cake' | 'chair' | 'couch' | 'potted plant' + | 'bed' | 'dining table' | 'toilet' | 'tv' | 'laptop' | 'mouse' | 'remote' | 'keyboard' | 'cell phone' | 'microwave' | 'oven' | 'toaster' | 'sink' | 'refrigerator' | 'book' + | 'clock' | 'vase' | 'scissors' | 'teddy bear' | 'hair drier' | 'toothbrush'; + /** Object results */ export interface ObjectResult { /** object id */ @@ -129,7 +155,7 @@ export interface ObjectResult { /** detected object class id */ class: number, /** detected object class name */ - label: string, + label: ObjectType, /** detected object box */ box: Box, /** detected object box normalized to 0..1 */ diff --git a/src/util/interpolate.ts b/src/util/interpolate.ts index c29c9f9f..ebae4a08 100644 --- a/src/util/interpolate.ts +++ b/src/util/interpolate.ts @@ -2,7 +2,7 @@ * Results interpolation for smoothening of video detection results inbetween detected frames */ -import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point } from '../result'; +import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point, BodyLandmark, BodyAnnotation } from '../result'; import type { Config } from '../config'; import * as moveNetCoords from '../body/movenetcoords'; @@ -46,7 +46,7 @@ export function calc(newResult: Result, config: Config): Result { const keypoints = (newResult.body[i].keypoints // update keypoints .map((newKpt, j) => ({ score: newKpt.score, - part: newKpt.part, + part: newKpt.part as BodyLandmark, position: [ bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[0] || 0) + (newKpt.position[0] || 0)) / bufferedFactor : newKpt.position[0], bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[1] || 0) + (newKpt.position[1] || 0)) / bufferedFactor : newKpt.position[1], @@ -57,9 +57,9 @@ export function calc(newResult: Result, config: Config): Result { bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1], bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2], ], - }))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>; + }))) as Array<{ score: number, part: BodyLandmark, position: [number, number, number?], positionRaw: [number, number, number?] }>; - const annotations: Record = {}; // recreate annotations + const annotations: Record = {} as Record; // recreate annotations let coords = { connected: {} }; if (config.body?.modelPath?.includes('efficientpose')) coords = efficientPoseCoords; else if (config.body?.modelPath?.includes('blazepose')) coords = blazePoseCoords;