diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index e49e39d3..4e652cc5 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -2,31 +2,42 @@ * Gesture detection module */ -import { Gesture } from '../result'; +import { Gesture, BodyGesture, FaceGesture, IrisGesture, FingerGesture, HandGesture, HandGestureType } from '../result'; export const body = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ body: number, gesture: string }> = []; + const gestures: Array<{ body: number; gesture: BodyGesture }> = []; for (let i = 0; i < res.length; i++) { // raising hands - const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist')); - const rightWrist = res[i].keypoints.find((a) => (a.part === 'rightWrist')); - const nose = res[i].keypoints.find((a) => (a.part === 'nose')); - if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'i give up' }); - else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise left hand' }); - else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise right hand' }); + const leftWrist = res[i].keypoints.find((a) => a.part === 'leftWrist'); + const rightWrist = res[i].keypoints.find((a) => a.part === 'rightWrist'); + const nose = res[i].keypoints.find((a) => a.part === 'nose'); + if ( + nose + && leftWrist + && rightWrist + && leftWrist.position.y < nose.position.y + && rightWrist.position.y < nose.position.y + ) gestures.push({ body: i, gesture: 'i give up' }); + else if (nose && leftWrist && leftWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise left hand' }); + else if (nose && rightWrist && rightWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise right hand' }); // leaning - const leftShoulder = res[i].keypoints.find((a) => (a.part === 'leftShoulder')); - const rightShoulder = res[i].keypoints.find((a) => (a.part === 'rightShoulder')); - if (leftShoulder && rightShoulder) gestures.push({ body: i, gesture: `leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}` }); + const leftShoulder = res[i].keypoints.find((a) => a.part === 'leftShoulder'); + const rightShoulder = res[i].keypoints.find((a) => a.part === 'rightShoulder'); + if (leftShoulder && rightShoulder) { + gestures.push({ + body: i, + gesture: `leaning ${leftShoulder.position.y > rightShoulder.position.y ? 'left' : 'right'}`, + }); + } } return gestures; }; export const face = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ face: number, gesture: string }> = []; + const gestures: Array<{ face: number; gesture: FaceGesture }> = []; for (let i = 0; i < res.length; i++) { if (res[i].mesh && res[i].mesh.length > 0) { const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2]; @@ -36,7 +47,10 @@ export const face = (res): Gesture[] => { if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' }); const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' }); - const mouthOpen = Math.min(100, 500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1]) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1])); + const mouthOpen = Math.min( + 100, + (500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1])) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]), + ); if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` }); const chinDepth = res[i].mesh[152][2]; if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` }); @@ -47,7 +61,7 @@ export const face = (res): Gesture[] => { export const iris = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ iris: number, gesture: string }> = []; + const gestures: Array<{ iris: number; gesture: IrisGesture }> = []; for (let i = 0; i < res.length; i++) { if (!res[i].annotations || !res[i].annotations.leftEyeIris || !res[i].annotations.rightEyeIris) continue; const sizeXLeft = res[i].annotations.leftEyeIris[3][0] - res[i].annotations.leftEyeIris[1][0]; @@ -85,16 +99,16 @@ export const iris = (res): Gesture[] => { export const hand = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ hand: number, gesture: string }> = []; + const gestures: Array<{ hand: number; gesture: HandGesture }> = []; for (let i = 0; i < res.length; i++) { - const fingers: Array<{ name: string, position: number }> = []; + const fingers: Array<{ name: FingerGesture; position: number }> = []; for (const [finger, pos] of Object.entries(res[i]['annotations'])) { - if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger + if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase() as FingerGesture, position: pos[0] }); // get tip of each finger } if (fingers && fingers.length > 0) { const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a)); - gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` }); + gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` as HandGestureType }); } } return gestures; diff --git a/src/persons.ts b/src/persons.ts index f3359d95..44f73cb5 100644 --- a/src/persons.ts +++ b/src/persons.ts @@ -1,14 +1,25 @@ /** * Module that analyzes existing results and recombines them into a unified person object */ - -import { Face, Body, Hand, Gesture, Person } from './result'; +import { Face, Body, Hand, Gesture, Person, IrisGesture, BodyGesture, HandGesture, FaceGesture } from './result'; export function join(faces: Array, bodies: Array, hands: Array, gestures: Array, shape: Array | undefined): Array { let id = 0; const persons: Array = []; for (const face of faces) { // person is defined primarily by face and then we append other objects as found - const person: Person = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] }; + const person: Person = { + id: id++, + face, + body: null, + hands: { left: null, right: null }, + gestures: { + face: [], + iris: [], + body: [], + hand: [], + }, + box: [0, 0, 0, 0], + }; for (const body of bodies) { if (face.box[0] > body.box[0] // x within body && face.box[0] < body.box[0] + body.box[2] @@ -34,11 +45,11 @@ export function join(faces: Array, bodies: Array, hands: Array } } for (const gesture of gestures) { // append all gestures according to ids - if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures?.push(gesture); - else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures?.push(gesture); - else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures?.push(gesture); - else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures?.push(gesture); - else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures?.push(gesture); + if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures.face.push(gesture.gesture as FaceGesture); + else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures.iris.push(gesture.gesture as IrisGesture); + else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures.body.push(gesture.gesture as BodyGesture); + else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures.hand.push(gesture.gesture as HandGesture); + else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures.hand.push(gesture.gesture as HandGesture); } // create new overarching box from all boxes beloning to person diff --git a/src/result.ts b/src/result.ts index 8dbf3084..bf34ce9d 100644 --- a/src/result.ts +++ b/src/result.ts @@ -4,6 +4,16 @@ import { Tensor } from './tfjs/types'; +/** + * @typedef Gender + */ +export type Gender = 'male' | 'female'; + +/** + * @typedef Emotion + */ +export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral'; + /** Face results * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models * Some values may be null if specific model is not enabled @@ -32,28 +42,28 @@ import { Tensor } from './tfjs/types'; * - tensor: face tensor as Tensor object which contains detected face */ export interface Face { - id: number - score: number, - boxScore: number, - faceScore: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - mesh: Array<[number, number, number]> - meshRaw: Array<[number, number, number]> - annotations: Record>, - age?: number, - gender?: string, - genderScore?: number, - emotion?: Array<{ score: number, emotion: string }>, - embedding?: Array, - iris?: number, + id: number; + score: number; + boxScore: number; + faceScore: number; + box: [number, number, number, number]; + boxRaw: [number, number, number, number]; + mesh: Array<[number, number, number]>; + meshRaw: Array<[number, number, number]>; + annotations: Record>; + age?: number; + gender?: Gender; + genderScore?: number; + emotion?: Array<{ score: number; emotion: Emotion }>; + embedding?: Array; + iris?: number; rotation?: { - angle: { roll: number, yaw: number, pitch: number }, - matrix: [number, number, number, number, number, number, number, number, number], - gaze: { bearing: number, strength: number }, - } + angle: { roll: number; yaw: number; pitch: number }; + matrix: [number, number, number, number, number, number, number, number, number]; + gaze: { bearing: number; strength: number }; + }; image?: Tensor; - tensor: Tensor, + tensor: Tensor; } /** Body results @@ -70,17 +80,17 @@ export interface Face { * - presence: body part presence value */ export interface Body { - id: number, - score: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], + id: number; + score: number; + box: [number, number, number, number]; + boxRaw: [number, number, number, number]; keypoints: Array<{ - part: string, - position: [number, number, number?], - positionRaw: [number, number, number?], - score: number, - presence?: number, - }> + part: string; + position: [number, number, number?]; + positionRaw: [number, number, number?]; + score: number; + presence?: number; + }>; } /** Hand results @@ -94,69 +104,144 @@ export interface Body { * - annotations: annotated landmarks for each hand part */ export interface Hand { - id: number, - score: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - keypoints: Array<[number, number, number]>, - annotations: Record>, + id: number; + score: number; + box: [number, number, number, number]; + boxRaw: [number, number, number, number]; + keypoints: Array<[number, number, number]>; + annotations: Record>; } /** Object results -* -* Array of individual results with one object per detected gesture -* Each result has: -* - id: object id number -* - score as value -* - label as detected class name -* - box: bounding box: x, y, width, height normalized to input image resolution -* - boxRaw: bounding box: x, y, width, height normalized to 0..1 -* - center: optional center point as array of [x, y], normalized to image resolution -* - centerRaw: optional center point as array of [x, y], normalized to range 0..1 -*/ -export interface Item { - id: number, - score: number, - class: number, - label: string, - box: [number, number, number, number], - boxRaw: [number, number, number, number], -} - -/** Gesture results - * @typedef Gesture Type * * Array of individual results with one object per detected gesture * Each result has: - * - part: part name and number where gesture was detected: face, iris, body, hand - * - gesture: gesture detected + * - id: object id number + * - score as value + * - label as detected class name + * - box: bounding box: x, y, width, height normalized to input image resolution + * - boxRaw: bounding box: x, y, width, height normalized to 0..1 + * - center: optional center point as array of [x, y], normalized to image resolution + * - centerRaw: optional center point as array of [x, y], normalized to range 0..1 */ +export interface Item { + id: number; + score: number; + class: number; + label: string; + box: [number, number, number, number]; + boxRaw: [number, number, number, number]; +} + +/** + * @typedef FaceGesture + */ +export type FaceGesture = + | `facing ${'left' | 'center' | 'right'}` + | `blink ${'left' | 'right'} eye` + | `mouth ${number}% open` + | `head ${'up' | 'down'}`; + +/** + * @typedef IrisGesture + */ +export type IrisGesture = + | 'facing center' + | `looking ${'left' | 'right' | 'up' | 'down'}` + | 'looking center'; + +/** + * @typedef BodyGesture + */ +export type BodyGesture = + | `leaning ${'left' | 'right'}` + | `raise ${'left' | 'right'} hand` + | 'i give up'; + +/** +* @typedef FingerThumb +*/ +export type FingerThumb = 'thumb'; + +/** +* @typedef FingerIndex +*/ +export type FingerIndex = 'indexfinger'; + +/** +* @typedef FingerMiddle +*/ +export type FingerMiddle = 'middlefinger'; + +/** +* @typedef FingerRing +*/ +export type FingerRing = 'ringfinger'; + +/** +* @typedef FingerPinky +*/ +export type FingerPinky = 'pinky'; + +/** +* @typedef FingerGesture +*/ +export type FingerGesture = FingerThumb | FingerIndex | FingerMiddle | FingerRing | FingerPinky; + +/** +* @typedef HandGestureType +*/ +export type HandGestureType = `${T} forward ${Exclude} up` + +/** + * @typedef HandGesture + */ +export type HandGesture = + | HandGestureType + | HandGestureType + | HandGestureType + | HandGestureType + | HandGestureType + +/** Gesture results +* @typedef Gesture Type +* +* Array of individual results with one object per detected gesture +* Each result has: +* - part: part name and number where gesture was detected: face, iris, body, hand +* - gesture: gesture detected +*/ export type Gesture = - { 'face': number, gesture: string } - | { 'iris': number, gesture: string } - | { 'body': number, gesture: string } - | { 'hand': number, gesture: string } + | { face: number; gesture: FaceGesture } + | { iris: number; gesture: IrisGesture } + | { body: number; gesture: BodyGesture } + | { hand: number; gesture: HandGesture }; /** Person getter -* @interface Person Interface -* -* Each result has: -* - id: person id -* - face: face object -* - body: body object -* - hands: array of hand objects -* - gestures: array of gestures -* - box: bounding box: x, y, width, height normalized to input image resolution -* - boxRaw: bounding box: x, y, width, height normalized to 0..1 -*/ + * @interface Person Interface + * + * Each result has: + * - id: person id + * - face: face object + * - body: body object + * - hands: hands object + * - gestures: gestures object + * - box: bounding box: x, y, width, height normalized to input image resolution + * - boxRaw: bounding box: x, y, width, height normalized to 0..1 + */ export interface Person { - id: number, - face: Face, - body: Body | null, - hands: { left: Hand | null, right: Hand | null }, - gestures: Array, - box: [number, number, number, number], - boxRaw?: [number, number, number, number], + id: number; + face: Face; + body: Body | null; + hands: { left: Hand | null; right: Hand | null }; + gestures: { + face: FaceGesture[]; + iris: IrisGesture[]; + body: BodyGesture[]; + hand: HandGesture[]; + }; + box: [number, number, number, number]; + boxRaw?: [number, number, number, number]; } /** @@ -166,21 +251,21 @@ export interface Person { */ export interface Result { /** {@link Face}: detection & analysis results */ - face: Array, + face: Array; /** {@link Body}: detection & analysis results */ - body: Array, + body: Array; /** {@link Hand}: detection & analysis results */ - hand: Array, + hand: Array; /** {@link Gesture}: detection & analysis results */ - gesture: Array, + gesture: Array; /** {@link Object}: detection & analysis results */ - object: Array + object: Array; /** global performance object with timing values for each operation */ - performance: Record, + performance: Record; /** optional processed canvas that can be used to draw input on screen */ - canvas?: OffscreenCanvas | HTMLCanvasElement, + canvas?: OffscreenCanvas | HTMLCanvasElement; /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ - readonly timestamp: number, + readonly timestamp: number; /** getter property that returns unified persons object */ - persons: Array, + persons: Array; }