ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

strong type for string enums

pull/280/head
Vladimir Mandic 2021-12-15 09:26:32 -05:00
parent b5862fb6a2
commit 8360cdb2ce
19 changed files with 93 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -9,8 +9,9 @@
## Changelog
### **HEAD -> main** 2021/12/13 mandic00@live.com
### **HEAD -> main** 2021/12/14 mandic00@live.com
- rebuild
- fix node detection in electron environment
### **2.5.5** 2021/12/01 mandic00@live.com

2
TODO.md
View File

@ -10,7 +10,6 @@
- Advanced histogram equalization: Adaptive, Contrast Limited, CLAHE
- TFLite models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
- Body segmentation: `robust-video-matting`
- TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
<br><hr><br>
@ -57,3 +56,4 @@ Other:
- Fix face detect box scale and rotation
- Fix body interpolation
- Updated `blazepose` implementation
- Strong typing for all string enums in `config` and `results`

6
src/body/blazepose.ts
View File

@ -5,7 +5,7 @@
import * as tf from '../../dist/tfjs.esm.js';
import { constants } from '../tfjs/constants';
import { log, join, now } from '../util/util';
import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
import type { BodyKeypoint, BodyResult, BodyLandmark, Box, Point, BodyAnnotation } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import * as coords from './blazeposecoords';
@ -144,13 +144,13 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
const adjScore = Math.trunc(100 * score * presence * poseScore) / 100;
const positionRaw: Point = [points[depth * i + 0] / inputSize.landmarks[0], points[depth * i + 1] / inputSize.landmarks[1], points[depth * i + 2] + 0];
const position: Point = [Math.trunc(outputSize[0] * positionRaw[0]), Math.trunc(outputSize[1] * positionRaw[1]), positionRaw[2] as number];
keypointsRelative.push({ part: coords.kpt[i], positionRaw, position, score: adjScore });
keypointsRelative.push({ part: coords.kpt[i] as BodyLandmark, positionRaw, position, score: adjScore });
}
if (poseScore < (config.body.minConfidence || 0)) return null;
const keypoints: Array<BodyKeypoint> = rescaleKeypoints(keypointsRelative, outputSize); // keypoints were relative to input image which is padded
const kpts = keypoints.map((k) => k.position);
const boxes = box.calc(kpts, [outputSize[0], outputSize[1]]); // now find boxes based on rescaled keypoints
const annotations: Record<string, Point[][]> = {};
const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>;
for (const [name, indexes] of Object.entries(coords.connected)) {
const pt: Array<Point[]> = [];
for (let i = 0; i < indexes.length - 1; i++) {

6
src/body/efficientpose.ts
View File

@ -8,14 +8,14 @@ import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import * as coords from './efficientposecoords';
import { constants } from '../tfjs/constants';
import type { BodyResult, Point } from '../result';
import type { BodyResult, Point, BodyLandmark, BodyAnnotation } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
let model: GraphModel | null;
let lastTime = 0;
const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} };
const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} as Record<BodyAnnotation, Point[][]> };
// const keypoints: Array<BodyKeypoint> = [];
// let box: Box = [0, 0, 0, 0];
@ -88,7 +88,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
if (partScore > (config.body?.minConfidence || 0)) {
cache.keypoints.push({
score: Math.round(100 * partScore) / 100,
part: coords.kpt[id],
part: coords.kpt[id] as BodyLandmark,
positionRaw: [ // normalized to 0..1
// @ts-ignore model is not undefined here
x / model.inputs[0].shape[2], y / model.inputs[0].shape[1],

8
src/body/movenet.ts
View File

@ -9,7 +9,7 @@ import * as box from '../util/box';
import * as tf from '../../dist/tfjs.esm.js';
import * as coords from './movenetcoords';
import * as fix from './movenetfix';
import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
import type { BodyKeypoint, BodyResult, BodyLandmark, BodyAnnotation, Box, Point } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { fakeOps } from '../tfjs/backend';
@ -52,7 +52,7 @@ async function parseSinglePose(res, config, image) {
const positionRaw: Point = [kpt[id][1], kpt[id][0]];
keypoints.push({
score: Math.round(100 * score) / 100,
part: coords.kpt[id],
part: coords.kpt[id] as BodyLandmark,
positionRaw,
position: [ // normalized to input image size
Math.round((image.shape[2] || 0) * positionRaw[0]),
@ -92,7 +92,7 @@ async function parseMultiPose(res, config, image) {
if (score > config.body.minConfidence) {
const positionRaw: Point = [kpt[3 * i + 1], kpt[3 * i + 0]];
keypoints.push({
part: coords.kpt[i],
part: coords.kpt[i] as BodyLandmark,
score: Math.round(100 * score) / 100,
positionRaw,
position: [Math.round((image.shape[2] || 0) * positionRaw[0]), Math.round((image.shape[1] || 0) * positionRaw[1])],
@ -103,7 +103,7 @@ async function parseMultiPose(res, config, image) {
// movenet-multipose has built-in box details
// const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
// const box: Box = [Math.trunc(boxRaw[0] * (image.shape[2] || 0)), Math.trunc(boxRaw[1] * (image.shape[1] || 0)), Math.trunc(boxRaw[2] * (image.shape[2] || 0)), Math.trunc(boxRaw[3] * (image.shape[1] || 0))];
const annotations: Record<string, Point[][]> = {};
const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>;
for (const [name, indexes] of Object.entries(coords.connected)) {
const pt: Array<Point[]> = [];
for (let i = 0; i < indexes.length - 1; i++) {

5
src/body/posenet.ts
View File

@ -6,7 +6,7 @@
import { log, join } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import type { BodyResult, Box } from '../result';
import type { BodyResult, BodyLandmark, Box } from '../result';
import type { Tensor, GraphModel } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
@ -14,7 +14,6 @@ import * as utils from './posenetutils';
let model: GraphModel;
const poseNetOutputs = ['MobilenetV1/offset_2/BiasAdd'/* offsets */, 'MobilenetV1/heatmap_2/BiasAdd'/* heatmapScores */, 'MobilenetV1/displacement_fwd_2/BiasAdd'/* displacementFwd */, 'MobilenetV1/displacement_bwd_2/BiasAdd'/* displacementBwd */];
const localMaximumRadius = 1;
const outputStride = 16;
const squaredNmsRadius = 50 ** 2;
@ -59,7 +58,7 @@ export function decodePose(root, scores, offsets, displacementsFwd, displacement
const rootPoint = utils.getImageCoords(root.part, outputStride, offsets);
keypoints[root.part.id] = {
score: root.score,
part: utils.partNames[root.part.id],
part: utils.partNames[root.part.id] as BodyLandmark,
position: rootPoint,
};
// Decode the part positions upwards in the tree, following the backward displacements.

13
src/body/posenetutils.ts
View File

@ -3,7 +3,7 @@
* See `posenet.ts` for entry point
*/
import type { BodyResult } from '../result';
import type { Point, BodyResult, BodyAnnotation, BodyLandmark } from '../result';
export const partNames = [
'nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftShoulder',
@ -71,17 +71,18 @@ export function getBoundingBox(keypoints): [number, number, number, number] {
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]): Array<BodyResult> {
const scaleY = height / inputResolutionHeight;
const scaleX = width / inputResolutionWidth;
const scalePose = (pose, i) => ({
const scalePose = (pose, i): BodyResult => ({
id: i,
score: pose.score,
boxRaw: [pose.box[0] / inputResolutionWidth, pose.box[1] / inputResolutionHeight, pose.box[2] / inputResolutionWidth, pose.box[3] / inputResolutionHeight],
box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)],
keypoints: pose.keypoints.map(({ score, part, position }) => ({
score,
part,
position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)],
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight],
score: score as number,
part: part as BodyLandmark,
position: [Math.trunc(position.x * scaleX), Math.trunc(position.y * scaleY)] as Point,
positionRaw: [position.x / inputResolutionHeight, position.y / inputResolutionHeight] as Point,
})),
annotations: {} as Record<BodyAnnotation, Point[][]>,
});
const scaledPoses = poses.map((pose, i) => scalePose(pose, i));
return scaledPoses;

1
src/config.ts
View File

@ -241,7 +241,6 @@ export interface Config {
* default: `full`
*/
warmup: '' | 'none' | 'face' | 'full' | 'body',
// warmup: string;
/** Base model path (typically starting with file://, http:// or https://) for all models
* - individual modelPath values are relative to this path

1
src/exports.ts
View File

@ -11,6 +11,7 @@ export type { Box, Point } from './result';
export type { Models } from './models';
export type { Env } from './util/env';
export type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
export type { Emotion, Finger, FingerCurl, FingerDirection, HandType, Gender, Race, FaceLandmark, BodyLandmark, BodyAnnotation, ObjectType } from './result';
export { env } from './util/env';
/** Events dispatched by `human.events`

6
src/face/face.ts
View File

@ -162,9 +162,9 @@ export const detectFace = async (instance: Human /* instance of human */, input:
// calculate iris distance
// iris: array[ center, left, top, right, bottom]
if (!instance.config.face.iris?.enabled && faces[i]?.annotations?.leftEyeIris && faces[i]?.annotations?.rightEyeIris) {
delete faces[i].annotations.leftEyeIris;
delete faces[i].annotations.rightEyeIris;
if (!instance.config.face.iris?.enabled) {
// if (faces[i]?.annotations?.leftEyeIris) delete faces[i].annotations.leftEyeIris;
// if (faces[i]?.annotations?.rightEyeIris) delete faces[i].annotations.rightEyeIris;
}
const irisSize = (faces[i].annotations && faces[i].annotations.leftEyeIris && faces[i].annotations.leftEyeIris[0] && faces[i].annotations.rightEyeIris && faces[i].annotations.rightEyeIris[0]
&& (faces[i].annotations.leftEyeIris.length > 0) && (faces[i].annotations.rightEyeIris.length > 0)

4
src/face/facemesh.ts
View File

@ -16,7 +16,7 @@ import * as iris from './iris';
import { histogramEqualization } from '../image/enhance';
import { env } from '../util/env';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { FaceResult, Point } from '../result';
import type { FaceResult, FaceLandmark, Point } from '../result';
import type { Config } from '../config';
type DetectBox = { startPoint: Point, endPoint: Point, landmarks: Array<Point>, confidence: number };
@ -62,7 +62,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
score: 0,
boxScore: 0,
faceScore: 0,
annotations: {},
annotations: {} as Record<FaceLandmark, Point[]>,
};
// optional rotation correction based on detector data only if mesh is disabled otherwise perform it later when we have more accurate mesh data. if no rotation correction this function performs crop

9
src/gear/emotion.ts
View File

@ -4,6 +4,7 @@
* [**Oarriaga**](https://github.com/oarriaga/face_classification)
*/
import type { Emotion } from '../result';
import { log, join, now } from '../util/util';
import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types';
@ -13,7 +14,7 @@ import { constants } from '../tfjs/constants';
const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
let model: GraphModel | null;
const last: Array<Array<{ score: number, emotion: string }>> = [];
const last: Array<Array<{ score: number, emotion: Emotion }>> = [];
let lastCount = 0;
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER;
@ -28,7 +29,7 @@ export async function load(config: Config): Promise<GraphModel> {
return model;
}
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<Array<{ score: number, emotion: string }>> {
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<Array<{ score: number, emotion: Emotion }>> {
if (!model) return [];
const skipFrame = skipped < (config.face.emotion?.skipFrames || 0);
const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime);
@ -38,7 +39,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count:
}
skipped = 0;
return new Promise(async (resolve) => {
const obj: Array<{ score: number, emotion: string }> = [];
const obj: Array<{ score: number, emotion: Emotion }> = [];
if (config.face.emotion?.enabled) {
const t: Record<string, Tensor> = {};
const inputSize = model?.inputs[0].shape ? model.inputs[0].shape[2] : 0;
@ -59,7 +60,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count:
lastTime = now();
const data = await t.emotion.data();
for (let i = 0; i < data.length; i++) {
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] as Emotion });
}
obj.sort((a, b) => b.score - a.score);
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));

5
src/gear/gear.ts
View File

@ -6,11 +6,12 @@
import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import type { Gender, Race } from '../result';
import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types';
import { env } from '../util/env';
type GearType = { age: number, gender: string, genderScore: number, race: Array<{ score: number, race: string }> }
type GearType = { age: number, gender: Gender, genderScore: number, race: Array<{ score: number, race: Race }> }
let model: GraphModel | null;
const last: Array<GearType> = [];
const raceNames = ['white', 'black', 'asian', 'indian', 'other'];
@ -53,7 +54,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis
obj.genderScore = Math.round(100 * (gender[0] > gender[1] ? gender[0] : gender[1])) / 100;
const race = await t.race.data();
for (let i = 0; i < race.length; i++) {
if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] });
if (race[i] > (config.face['gear']?.minConfidence || 0.2)) obj.race.push({ score: Math.round(100 * race[i]) / 100, race: raceNames[i] as Race });
}
obj.race.sort((a, b) => b.score - a.score);
// {0: 'Below20', 1: '21-25', 2: '26-30', 3: '31-40',4: '41-50', 5: '51-60', 6: 'Above60'}

7
src/gear/ssrnet-gender.ts
View File

@ -7,12 +7,13 @@
import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import { constants } from '../tfjs/constants';
import type { Gender } from '../result';
import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types';
import { env } from '../util/env';
let model: GraphModel | null;
const last: Array<{ gender: string, genderScore: number }> = [];
const last: Array<{ gender: Gender, genderScore: number }> = [];
let lastCount = 0;
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER;
@ -32,7 +33,7 @@ export async function load(config: Config | any) {
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: string, genderScore: number }> {
export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: Gender, genderScore: number }> {
if (!model) return { gender: 'unknown', genderScore: 0 };
const skipFrame = skipped < (config.face['ssrnet']?.skipFrames || 0);
const skipTime = (config.face['ssrnet']?.skipTime || 0) > (now() - lastTime);
@ -54,7 +55,7 @@ export async function predict(image: Tensor, config: Config, idx, count): Promis
const normalize = tf.mul(tf.sub(grayscale, constants.tf05), 2); // range grayscale:-1..1
return normalize;
});
const obj = { gender: '', genderScore: 0 };
const obj: { gender: Gender, genderScore: number } = { gender: 'unknown', genderScore: 0 };
if (config.face['ssrnet'].enabled) t.gender = model.execute(t.enhance) as Tensor;
const data = await t.gender.data();
obj.gender = data[0] > data[1] ? 'female' : 'male'; // returns two values 0..1, bigger one is prediction

6
src/hand/handtrack.ts
View File

@ -9,7 +9,7 @@
import { log, join, now } from '../util/util';
import * as box from '../util/box';
import * as tf from '../../dist/tfjs.esm.js';
import type { HandResult, Box, Point } from '../result';
import type { HandResult, HandType, Box, Point } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
@ -39,7 +39,7 @@ type HandDetectResult = {
box: Box,
boxRaw: Box,
boxCrop: Box,
label: string,
label: HandType,
}
const cache: {
@ -129,7 +129,7 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
const boxCrop: Box = box.crop(boxRaw); // crop box is based on raw box
const boxFull: Box = [Math.trunc(boxData[0] * outputSize[0]), Math.trunc(boxData[1] * outputSize[1]), Math.trunc(boxData[2] * outputSize[0]), Math.trunc(boxData[3] * outputSize[1])];
const score = scores[nmsIndex];
const label = classes[classNum[nmsIndex]];
const label = classes[classNum[nmsIndex]] as HandType;
const hand: HandDetectResult = { id: id++, score, box: boxFull, boxRaw, boxCrop, label };
hands.push(hand);
}

4
src/object/centernet.ts
View File

@ -7,7 +7,7 @@
import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import { labels } from './labels';
import type { ObjectResult, Box } from '../result';
import type { ObjectResult, ObjectType, Box } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
@ -49,7 +49,7 @@ async function process(res: Tensor | null, outputShape, config: Config) {
for (const id of Array.from(nms)) {
const score = Math.trunc(100 * detections[0][id][4]) / 100;
const classVal = detections[0][id][5];
const label = labels[classVal].label;
const label = labels[classVal].label as ObjectType;
const [x, y] = [
detections[0][id][0] / inputSize,
detections[0][id][1] / inputSize,

4
src/object/nanodet.ts
View File

@ -8,7 +8,7 @@ import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js';
import { constants } from '../tfjs/constants';
import { labels } from './labels';
import type { ObjectResult, Box } from '../result';
import type { ObjectResult, ObjectType, Box } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config';
import { env } from '../util/env';
@ -72,7 +72,7 @@ async function process(res, inputSize, outputShape, config) {
// strideSize,
score: Math.round(100 * score) / 100,
class: j + 1,
label: labels[j].label,
label: labels[j].label as ObjectType,
// center: [Math.trunc(outputShape[0] * cx), Math.trunc(outputShape[1] * cy)],
// centerRaw: [cx, cy],
box: box.map((a) => Math.trunc(a)) as Box,

58
src/result.ts
View File

@ -11,6 +11,15 @@ export type Box = [number, number, number, number];
/** generic point as [x, y, z?] */
export type Point = [number, number, number?];
export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral';
export type Gender = 'male' | 'female' | 'unknown';
export type Race = 'white' | 'black' | 'asian' | 'indian' | 'other';
export type FaceLandmark = 'leftEye' | 'rightEye' | 'nose' | 'mouth' | 'leftEar' | 'rightEar' | 'symmetryLine' | 'silhouette'
| 'lipsUpperOuter' | 'lipsLowerOuter' | 'lipsUpperInner' | 'lipsLowerInner'
| 'rightEyeUpper0' | 'rightEyeLower0' | 'rightEyeUpper1' | 'rightEyeLower1' | 'rightEyeUpper2' | 'rightEyeLower2' | 'rightEyeLower3' | 'rightEyebrowUpper' | 'rightEyebrowLower' | 'rightEyeIris'
| 'leftEyeUpper0' | 'leftEyeLower0' | 'leftEyeUpper1' | 'leftEyeLower1' | 'leftEyeUpper2' | 'leftEyeLower2' | 'leftEyeLower3' | 'leftEyebrowUpper' | 'leftEyebrowLower' | 'leftEyeIris'
| 'midwayBetweenEyes' | 'noseTip' | 'noseBottom' | 'noseRightCorner' | 'noseLeftCorner' | 'rightCheek' | 'leftCheek';
/** Face results
* - Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
* - Some values may be null if specific model is not enabled
@ -33,17 +42,17 @@ export interface FaceResult {
/** detected face mesh normalized to 0..1 */
meshRaw: Array<Point>
/** mesh keypoints combined into annotated results */
annotations: Record<string, Point[]>,
annotations: Record<FaceLandmark, Point[]>,
/** detected age */
age?: number,
/** detected gender */
gender?: string,
gender?: Gender,
/** gender detection score */
genderScore?: number,
/** detected emotions */
emotion?: Array<{ score: number, emotion: string }>,
emotion?: Array<{ score: number, emotion: Emotion }>,
/** detected race */
race?: Array<{ score: number, race: string }>,
race?: Array<{ score: number, race: Race }>,
/** face descriptor */
embedding?: Array<number>,
/** face iris distance from camera */
@ -62,10 +71,21 @@ export interface FaceResult {
tensor?: Tensor,
}
export type BodyLandmarkPoseNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
export type BodyLandmarkMoveNet = 'nose' | 'leftEye' | 'rightEye' | 'leftEar' | 'rightEar' | 'leftShoulder' | 'rightShoulder' | 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle';
export type BodyLandmarkEfficientNet = 'head' | 'neck' | 'rightShoulder' | 'rightElbow' | 'rightWrist' | 'chest' | 'leftShoulder' | 'leftElbow' | 'leftWrist' | 'bodyCenter' | 'rightHip' | 'rightKnee' | 'rightAnkle' | 'leftHip' | 'leftKnee' | 'leftAnkle';
export type BodyLandmarkBlazePose = 'nose' | 'leftEyeInside' | 'leftEye' | 'leftEyeOutside' | 'rightEyeInside' | 'rightEye' | 'rightEyeOutside' | 'leftEar' | 'rightEar' | 'leftMouth' | 'rightMouth' | 'leftShoulder' | 'rightShoulder'
| 'leftElbow' | 'rightElbow' | 'leftWrist' | 'rightWrist' | 'leftPinky' | 'rightPinky' | 'leftIndex' | 'rightIndex' | 'leftThumb' | 'rightThumb' | 'leftHip' | 'rightHip' | 'leftKnee' | 'rightKnee' | 'leftAnkle' | 'rightAnkle'
| 'leftHeel' | 'rightHeel' | 'leftFoot' | 'rightFoot' | 'bodyCenter' | 'bodyTop' | 'leftPalm' | 'leftHand' | 'rightPalm' | 'rightHand';
export type BodyLandmark = BodyLandmarkPoseNet | BodyLandmarkMoveNet | BodyLandmarkEfficientNet | BodyLandmarkBlazePose;
export type BodyAnnotationBlazePose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'leftEye' | 'rightEye' | 'mouth';
export type BodyAnnotationEfficientPose = 'leftLeg' | 'rightLeg' | 'torso' | 'leftArm' | 'rightArm' | 'head';
export type BodyAnnotation = BodyAnnotationBlazePose | BodyAnnotationEfficientPose;
/** Body Result keypoints */
export interface BodyKeypoint {
/** body part name */
part: string,
part: BodyLandmark,
/** body part position */
position: Point,
/** body part position normalized to 0..1 */
@ -87,9 +107,14 @@ export interface BodyResult {
/** detected body keypoints */
keypoints: Array<BodyKeypoint>
/** detected body keypoints combined into annotated parts */
annotations: Record<string, Array<Point[]>>,
annotations: Record<BodyAnnotation, Point[][]>,
}
export type HandType = 'hand' | 'fist' | 'pinch' | 'point' | 'face' | 'tip' | 'pinchtip';
export type Finger = 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm';
export type FingerCurl = 'none' | 'half' | 'full';
export type FingerDirection = 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft';
/** Hand results */
export interface HandResult {
/** hand id */
@ -107,19 +132,20 @@ export interface HandResult {
/** detected hand keypoints */
keypoints: Array<Point>,
/** detected hand class */
label: string,
label: HandType,
/** detected hand keypoints combined into annotated parts */
annotations: Record<
'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
Array<Point>
>,
annotations: Record<Finger, Array<Point>>,
/** detected hand parts annotated with part gestures */
landmarks: Record<
'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
{ curl: 'none' | 'half' | 'full', direction: 'verticalUp' | 'verticalDown' | 'horizontalLeft' | 'horizontalRight' | 'diagonalUpRight' | 'diagonalUpLeft' | 'diagonalDownRight' | 'diagonalDownLeft' }
>,
landmarks: Record<Finger, { curl: FingerCurl, direction: FingerDirection }>,
}
export type ObjectType = 'person' | 'bicycle' | 'car' | 'motorcycle' | 'airplane' | 'bus' | 'train' | 'truck' | 'boat' | 'traffic light' | 'fire hydrant' | 'stop sign' | 'parking meter'
| 'bench' | 'bird' | 'cat' | 'dog' | 'horse' | 'sheep' | 'cow' | 'elephant' | 'bear' | 'zebra' | 'giraffe' | 'backpack' | 'umbrella' | 'handbag' | 'tie' | 'suitcase' | 'frisbee'
| 'skis' | 'snowboard' | 'sports ball' | 'kite' | 'baseball bat' | 'baseball glove' | 'skateboard' | 'surfboard' | 'tennis racket' | 'bottle' | 'wine glass' | 'cup' | 'fork'
| 'knife' | 'spoon' | 'bowl' | 'banana' | 'apple' | 'sandwich' | 'orange' | 'broccoli' | 'carrot' | 'hot dog' | 'pizza' | 'donut' | 'cake' | 'chair' | 'couch' | 'potted plant'
| 'bed' | 'dining table' | 'toilet' | 'tv' | 'laptop' | 'mouse' | 'remote' | 'keyboard' | 'cell phone' | 'microwave' | 'oven' | 'toaster' | 'sink' | 'refrigerator' | 'book'
| 'clock' | 'vase' | 'scissors' | 'teddy bear' | 'hair drier' | 'toothbrush';
/** Object results */
export interface ObjectResult {
/** object id */
@ -129,7 +155,7 @@ export interface ObjectResult {
/** detected object class id */
class: number,
/** detected object class name */
label: string,
label: ObjectType,
/** detected object box */
box: Box,
/** detected object box normalized to 0..1 */

8
src/util/interpolate.ts
View File

@ -2,7 +2,7 @@
* Results interpolation for smoothening of video detection results inbetween detected frames
*/
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point } from '../result';
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point, BodyLandmark, BodyAnnotation } from '../result';
import type { Config } from '../config';
import * as moveNetCoords from '../body/movenetcoords';
@ -46,7 +46,7 @@ export function calc(newResult: Result, config: Config): Result {
const keypoints = (newResult.body[i].keypoints // update keypoints
.map((newKpt, j) => ({
score: newKpt.score,
part: newKpt.part,
part: newKpt.part as BodyLandmark,
position: [
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[0] || 0) + (newKpt.position[0] || 0)) / bufferedFactor : newKpt.position[0],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].position[1] || 0) + (newKpt.position[1] || 0)) / bufferedFactor : newKpt.position[1],
@ -57,9 +57,9 @@ export function calc(newResult: Result, config: Config): Result {
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[1] || 0) + (newKpt.positionRaw[1] || 0)) / bufferedFactor : newKpt.position[1],
bufferedResult.body[i].keypoints[j] ? ((bufferedFactor - 1) * (bufferedResult.body[i].keypoints[j].positionRaw[2] || 0) + (newKpt.positionRaw[2] || 0)) / bufferedFactor : newKpt.position[2],
],
}))) as Array<{ score: number, part: string, position: [number, number, number?], positionRaw: [number, number, number?] }>;
}))) as Array<{ score: number, part: BodyLandmark, position: [number, number, number?], positionRaw: [number, number, number?] }>;
const annotations: Record<string, Point[][]> = {}; // recreate annotations
const annotations: Record<BodyAnnotation, Point[][]> = {} as Record<BodyAnnotation, Point[][]>; // recreate annotations
let coords = { connected: {} };
if (config.body?.modelPath?.includes('efficientpose')) coords = efficientPoseCoords;
else if (config.body?.modelPath?.includes('blazepose')) coords = blazePoseCoords;