ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve result types

pull/134/head
pg g 2021-07-06 16:38:41 +02:00
parent 0d58300990
commit aba23bbe0b
3 changed files with 227 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/gesture/gesture.ts
View File

@ -2,31 +2,42 @@
* Gesture detection module
*/
import { Gesture } from '../result';
import { Gesture, BodyGesture, FaceGesture, IrisGesture, FingerGesture, HandGesture, HandGestureType } from '../result';
export const body = (res): Gesture[] => {
if (!res) return [];
const gestures: Array<{ body: number, gesture: string }> = [];
const gestures: Array<{ body: number; gesture: BodyGesture }> = [];
for (let i = 0; i < res.length; i++) {
// raising hands
const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist'));
const rightWrist = res[i].keypoints.find((a) => (a.part === 'rightWrist'));
const nose = res[i].keypoints.find((a) => (a.part === 'nose'));
if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'i give up' });
else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise left hand' });
else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise right hand' });
const leftWrist = res[i].keypoints.find((a) => a.part === 'leftWrist');
const rightWrist = res[i].keypoints.find((a) => a.part === 'rightWrist');
const nose = res[i].keypoints.find((a) => a.part === 'nose');
if (
nose
&& leftWrist
&& rightWrist
&& leftWrist.position.y < nose.position.y
&& rightWrist.position.y < nose.position.y
) gestures.push({ body: i, gesture: 'i give up' });
else if (nose && leftWrist && leftWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise left hand' });
else if (nose && rightWrist && rightWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise right hand' });
// leaning
const leftShoulder = res[i].keypoints.find((a) => (a.part === 'leftShoulder'));
const rightShoulder = res[i].keypoints.find((a) => (a.part === 'rightShoulder'));
if (leftShoulder && rightShoulder) gestures.push({ body: i, gesture: `leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}` });
const leftShoulder = res[i].keypoints.find((a) => a.part === 'leftShoulder');
const rightShoulder = res[i].keypoints.find((a) => a.part === 'rightShoulder');
if (leftShoulder && rightShoulder) {
gestures.push({
body: i,
gesture: `leaning ${leftShoulder.position.y > rightShoulder.position.y ? 'left' : 'right'}`,
});
}
}
return gestures;
};
export const face = (res): Gesture[] => {
if (!res) return [];
const gestures: Array<{ face: number, gesture: string }> = [];
const gestures: Array<{ face: number; gesture: FaceGesture }> = [];
for (let i = 0; i < res.length; i++) {
if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2];
@ -36,7 +47,10 @@ export const face = (res): Gesture[] => {
if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' });
const mouthOpen = Math.min(100, 500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1]) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]));
const mouthOpen = Math.min(
100,
(500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1])) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]),
);
if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` });
const chinDepth = res[i].mesh[152][2];
if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` });
@ -47,7 +61,7 @@ export const face = (res): Gesture[] => {
export const iris = (res): Gesture[] => {
if (!res) return [];
const gestures: Array<{ iris: number, gesture: string }> = [];
const gestures: Array<{ iris: number; gesture: IrisGesture }> = [];
for (let i = 0; i < res.length; i++) {
if (!res[i].annotations || !res[i].annotations.leftEyeIris || !res[i].annotations.rightEyeIris) continue;
const sizeXLeft = res[i].annotations.leftEyeIris[3][0] - res[i].annotations.leftEyeIris[1][0];
@ -85,16 +99,16 @@ export const iris = (res): Gesture[] => {
export const hand = (res): Gesture[] => {
if (!res) return [];
const gestures: Array<{ hand: number, gesture: string }> = [];
const gestures: Array<{ hand: number; gesture: HandGesture }> = [];
for (let i = 0; i < res.length; i++) {
const fingers: Array<{ name: string, position: number }> = [];
const fingers: Array<{ name: FingerGesture; position: number }> = [];
for (const [finger, pos] of Object.entries(res[i]['annotations'])) {
if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger
if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase() as FingerGesture, position: pos[0] }); // get tip of each finger
}
if (fingers && fingers.length > 0) {
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` });
gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` as HandGestureType<typeof closest.name> });
}
}
return gestures;

27
src/persons.ts
View File

@ -1,14 +1,25 @@
/**
* Module that analyzes existing results and recombines them into a unified person object
*/
import { Face, Body, Hand, Gesture, Person } from './result';
import { Face, Body, Hand, Gesture, Person, IrisGesture, BodyGesture, HandGesture, FaceGesture } from './result';
export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>, gestures: Array<Gesture>, shape: Array<number> | undefined): Array<Person> {
let id = 0;
const persons: Array<Person> = [];
for (const face of faces) { // person is defined primarily by face and then we append other objects as found
const person: Person = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] };
const person: Person = {
id: id++,
face,
body: null,
hands: { left: null, right: null },
gestures: {
face: [],
iris: [],
body: [],
hand: [],
},
box: [0, 0, 0, 0],
};
for (const body of bodies) {
if (face.box[0] > body.box[0] // x within body
&& face.box[0] < body.box[0] + body.box[2]
@ -34,11 +45,11 @@ export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>
}
}
for (const gesture of gestures) { // append all gestures according to ids
if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures?.push(gesture);
else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures?.push(gesture);
else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures?.push(gesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures?.push(gesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures?.push(gesture);
if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures.face.push(gesture.gesture as FaceGesture);
else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures.iris.push(gesture.gesture as IrisGesture);
else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures.body.push(gesture.gesture as BodyGesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures.hand.push(gesture.gesture as HandGesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures.hand.push(gesture.gesture as HandGesture);
}
// create new overarching box from all boxes beloning to person

267
src/result.ts
View File

@ -4,6 +4,16 @@
import { Tensor } from './tfjs/types';
/**
* @typedef Gender
*/
export type Gender = 'male' | 'female';
/**
* @typedef Emotion
*/
export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral';
/** Face results
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
* Some values may be null if specific model is not enabled
@ -32,28 +42,28 @@ import { Tensor } from './tfjs/types';
* - tensor: face tensor as Tensor object which contains detected face
*/
export interface Face {
id: number
score: number,
boxScore: number,
faceScore: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
mesh: Array<[number, number, number]>
meshRaw: Array<[number, number, number]>
annotations: Record<string, Array<[number, number, number]>>,
age?: number,
gender?: string,
genderScore?: number,
emotion?: Array<{ score: number, emotion: string }>,
embedding?: Array<number>,
iris?: number,
id: number;
score: number;
boxScore: number;
faceScore: number;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
mesh: Array<[number, number, number]>;
meshRaw: Array<[number, number, number]>;
annotations: Record<string, Array<[number, number, number]>>;
age?: number;
gender?: Gender;
genderScore?: number;
emotion?: Array<{ score: number; emotion: Emotion }>;
embedding?: Array<number>;
iris?: number;
rotation?: {
angle: { roll: number, yaw: number, pitch: number },
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { bearing: number, strength: number },
}
angle: { roll: number; yaw: number; pitch: number };
matrix: [number, number, number, number, number, number, number, number, number];
gaze: { bearing: number; strength: number };
};
image?: Tensor;
tensor: Tensor,
tensor: Tensor;
}
/** Body results
@ -70,17 +80,17 @@ export interface Face {
* - presence: body part presence value
*/
export interface Body {
id: number,
score: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
id: number;
score: number;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
keypoints: Array<{
part: string,
position: [number, number, number?],
positionRaw: [number, number, number?],
score: number,
presence?: number,
}>
part: string;
position: [number, number, number?];
positionRaw: [number, number, number?];
score: number;
presence?: number;
}>;
}
/** Hand results
@ -94,69 +104,144 @@ export interface Body {
* - annotations: annotated landmarks for each hand part
*/
export interface Hand {
id: number,
score: number,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
keypoints: Array<[number, number, number]>,
annotations: Record<string, Array<[number, number, number]>>,
id: number;
score: number;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
keypoints: Array<[number, number, number]>;
annotations: Record<string, Array<[number, number, number]>>;
}
/** Object results
*
* Array of individual results with one object per detected gesture
* Each result has:
* - id: object id number
* - score as value
* - label as detected class name
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
* - center: optional center point as array of [x, y], normalized to image resolution
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
*/
export interface Item {
id: number,
score: number,
class: number,
label: string,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
}
/** Gesture results
* @typedef Gesture Type
*
* Array of individual results with one object per detected gesture
* Each result has:
* - part: part name and number where gesture was detected: face, iris, body, hand
* - gesture: gesture detected
* - id: object id number
* - score as value
* - label as detected class name
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
* - center: optional center point as array of [x, y], normalized to image resolution
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
*/
export interface Item {
id: number;
score: number;
class: number;
label: string;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
}
/**
* @typedef FaceGesture
*/
export type FaceGesture =
| `facing ${'left' | 'center' | 'right'}`
| `blink ${'left' | 'right'} eye`
| `mouth ${number}% open`
| `head ${'up' | 'down'}`;
/**
* @typedef IrisGesture
*/
export type IrisGesture =
| 'facing center'
| `looking ${'left' | 'right' | 'up' | 'down'}`
| 'looking center';
/**
* @typedef BodyGesture
*/
export type BodyGesture =
| `leaning ${'left' | 'right'}`
| `raise ${'left' | 'right'} hand`
| 'i give up';
/**
* @typedef FingerThumb
*/
export type FingerThumb = 'thumb';
/**
* @typedef FingerIndex
*/
export type FingerIndex = 'indexfinger';
/**
* @typedef FingerMiddle
*/
export type FingerMiddle = 'middlefinger';
/**
* @typedef FingerRing
*/
export type FingerRing = 'ringfinger';
/**
* @typedef FingerPinky
*/
export type FingerPinky = 'pinky';
/**
* @typedef FingerGesture
*/
export type FingerGesture = FingerThumb | FingerIndex | FingerMiddle | FingerRing | FingerPinky;
/**
* @typedef HandGestureType
*/
export type HandGestureType<T extends FingerGesture> = `${T} forward ${Exclude<FingerGesture, T>} up`
/**
* @typedef HandGesture
*/
export type HandGesture =
| HandGestureType<FingerThumb>
| HandGestureType<FingerIndex>
| HandGestureType<FingerMiddle>
| HandGestureType<FingerRing>
| HandGestureType<FingerPinky>
/** Gesture results
* @typedef Gesture Type
*
* Array of individual results with one object per detected gesture
* Each result has:
* - part: part name and number where gesture was detected: face, iris, body, hand
* - gesture: gesture detected
*/
export type Gesture =
{ 'face': number, gesture: string }
| { 'iris': number, gesture: string }
| { 'body': number, gesture: string }
| { 'hand': number, gesture: string }
| { face: number; gesture: FaceGesture }
| { iris: number; gesture: IrisGesture }
| { body: number; gesture: BodyGesture }
| { hand: number; gesture: HandGesture };
/** Person getter
* @interface Person Interface
*
* Each result has:
* - id: person id
* - face: face object
* - body: body object
* - hands: array of hand objects
* - gestures: array of gestures
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
*/
* @interface Person Interface
*
* Each result has:
* - id: person id
* - face: face object
* - body: body object
* - hands: hands object
* - gestures: gestures object
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
*/
export interface Person {
id: number,
face: Face,
body: Body | null,
hands: { left: Hand | null, right: Hand | null },
gestures: Array<Gesture>,
box: [number, number, number, number],
boxRaw?: [number, number, number, number],
id: number;
face: Face;
body: Body | null;
hands: { left: Hand | null; right: Hand | null };
gestures: {
face: FaceGesture[];
iris: IrisGesture[];
body: BodyGesture[];
hand: HandGesture[];
};
box: [number, number, number, number];
boxRaw?: [number, number, number, number];
}
/**
@ -166,21 +251,21 @@ export interface Person {
*/
export interface Result {
/** {@link Face}: detection & analysis results */
face: Array<Face>,
face: Array<Face>;
/** {@link Body}: detection & analysis results */
body: Array<Body>,
body: Array<Body>;
/** {@link Hand}: detection & analysis results */
hand: Array<Hand>,
hand: Array<Hand>;
/** {@link Gesture}: detection & analysis results */
gesture: Array<Gesture>,
gesture: Array<Gesture>;
/** {@link Object}: detection & analysis results */
object: Array<Item>
object: Array<Item>;
/** global performance object with timing values for each operation */
performance: Record<string, unknown>,
performance: Record<string, unknown>;
/** optional processed canvas that can be used to draw input on screen */
canvas?: OffscreenCanvas | HTMLCanvasElement,
canvas?: OffscreenCanvas | HTMLCanvasElement;
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
readonly timestamp: number,
readonly timestamp: number;
/** getter property that returns unified persons object */
persons: Array<Person>,
persons: Array<Person>;
}