ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve result types

pull/134/head
pg g 2021-07-06 16:38:41 +02:00
parent 0d58300990
commit aba23bbe0b
3 changed files with 227 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/gesture/gesture.ts
View File

@ -2,31 +2,42 @@
* Gesture detection module * Gesture detection module
*/ */
import { Gesture } from '../result'; import { Gesture, BodyGesture, FaceGesture, IrisGesture, FingerGesture, HandGesture, HandGestureType } from '../result';
export const body = (res): Gesture[] => { export const body = (res): Gesture[] => {
if (!res) return []; if (!res) return [];
const gestures: Array<{ body: number, gesture: string }> = []; const gestures: Array<{ body: number; gesture: BodyGesture }> = [];
for (let i = 0; i < res.length; i++) { for (let i = 0; i < res.length; i++) {
// raising hands // raising hands
const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist')); const leftWrist = res[i].keypoints.find((a) => a.part === 'leftWrist');
const rightWrist = res[i].keypoints.find((a) => (a.part === 'rightWrist')); const rightWrist = res[i].keypoints.find((a) => a.part === 'rightWrist');
const nose = res[i].keypoints.find((a) => (a.part === 'nose')); const nose = res[i].keypoints.find((a) => a.part === 'nose');
if (nose && leftWrist && rightWrist && (leftWrist.position.y < nose.position.y) && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'i give up' }); if (
else if (nose && leftWrist && (leftWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise left hand' }); nose
else if (nose && rightWrist && (rightWrist.position.y < nose.position.y)) gestures.push({ body: i, gesture: 'raise right hand' }); && leftWrist
&& rightWrist
&& leftWrist.position.y < nose.position.y
&& rightWrist.position.y < nose.position.y
) gestures.push({ body: i, gesture: 'i give up' });
else if (nose && leftWrist && leftWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise left hand' });
else if (nose && rightWrist && rightWrist.position.y < nose.position.y) gestures.push({ body: i, gesture: 'raise right hand' });
// leaning // leaning
const leftShoulder = res[i].keypoints.find((a) => (a.part === 'leftShoulder')); const leftShoulder = res[i].keypoints.find((a) => a.part === 'leftShoulder');
const rightShoulder = res[i].keypoints.find((a) => (a.part === 'rightShoulder')); const rightShoulder = res[i].keypoints.find((a) => a.part === 'rightShoulder');
if (leftShoulder && rightShoulder) gestures.push({ body: i, gesture: `leaning ${(leftShoulder.position.y > rightShoulder.position.y) ? 'left' : 'right'}` }); if (leftShoulder && rightShoulder) {
gestures.push({
body: i,
gesture: `leaning ${leftShoulder.position.y > rightShoulder.position.y ? 'left' : 'right'}`,
});
}
} }
return gestures; return gestures;
}; };
export const face = (res): Gesture[] => { export const face = (res): Gesture[] => {
if (!res) return []; if (!res) return [];
const gestures: Array<{ face: number, gesture: string }> = []; const gestures: Array<{ face: number; gesture: FaceGesture }> = [];
for (let i = 0; i < res.length; i++) { for (let i = 0; i < res.length; i++) {
if (res[i].mesh && res[i].mesh.length > 0) { if (res[i].mesh && res[i].mesh.length > 0) {
const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2]; const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2];
@ -36,7 +47,10 @@ export const face = (res): Gesture[] => {
if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' }); if (openLeft < 0.2) gestures.push({ face: i, gesture: 'blink left eye' });
const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord const openRight = Math.abs(res[i].mesh[145][1] - res[i].mesh[159][1]) / Math.abs(res[i].mesh[223][1] - res[i].mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' }); if (openRight < 0.2) gestures.push({ face: i, gesture: 'blink right eye' });
const mouthOpen = Math.min(100, 500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1]) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1])); const mouthOpen = Math.min(
100,
(500 * Math.abs(res[i].mesh[13][1] - res[i].mesh[14][1])) / Math.abs(res[i].mesh[10][1] - res[i].mesh[152][1]),
);
if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` }); if (mouthOpen > 10) gestures.push({ face: i, gesture: `mouth ${Math.trunc(mouthOpen)}% open` });
const chinDepth = res[i].mesh[152][2]; const chinDepth = res[i].mesh[152][2];
if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` }); if (Math.abs(chinDepth) > 10) gestures.push({ face: i, gesture: `head ${chinDepth < 0 ? 'up' : 'down'}` });
@ -47,7 +61,7 @@ export const face = (res): Gesture[] => {
export const iris = (res): Gesture[] => { export const iris = (res): Gesture[] => {
if (!res) return []; if (!res) return [];
const gestures: Array<{ iris: number, gesture: string }> = []; const gestures: Array<{ iris: number; gesture: IrisGesture }> = [];
for (let i = 0; i < res.length; i++) { for (let i = 0; i < res.length; i++) {
if (!res[i].annotations || !res[i].annotations.leftEyeIris || !res[i].annotations.rightEyeIris) continue; if (!res[i].annotations || !res[i].annotations.leftEyeIris || !res[i].annotations.rightEyeIris) continue;
const sizeXLeft = res[i].annotations.leftEyeIris[3][0] - res[i].annotations.leftEyeIris[1][0]; const sizeXLeft = res[i].annotations.leftEyeIris[3][0] - res[i].annotations.leftEyeIris[1][0];
@ -85,16 +99,16 @@ export const iris = (res): Gesture[] => {
export const hand = (res): Gesture[] => { export const hand = (res): Gesture[] => {
if (!res) return []; if (!res) return [];
const gestures: Array<{ hand: number, gesture: string }> = []; const gestures: Array<{ hand: number; gesture: HandGesture }> = [];
for (let i = 0; i < res.length; i++) { for (let i = 0; i < res.length; i++) {
const fingers: Array<{ name: string, position: number }> = []; const fingers: Array<{ name: FingerGesture; position: number }> = [];
for (const [finger, pos] of Object.entries(res[i]['annotations'])) { for (const [finger, pos] of Object.entries(res[i]['annotations'])) {
if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase() as FingerGesture, position: pos[0] }); // get tip of each finger
} }
if (fingers && fingers.length > 0) { if (fingers && fingers.length > 0) {
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a)); const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` }); gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` as HandGestureType<typeof closest.name> });
} }
} }
return gestures; return gestures;

27
src/persons.ts
View File

@ -1,14 +1,25 @@
/** /**
* Module that analyzes existing results and recombines them into a unified person object * Module that analyzes existing results and recombines them into a unified person object
*/ */
import { Face, Body, Hand, Gesture, Person, IrisGesture, BodyGesture, HandGesture, FaceGesture } from './result';
import { Face, Body, Hand, Gesture, Person } from './result';
export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>, gestures: Array<Gesture>, shape: Array<number> | undefined): Array<Person> { export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>, gestures: Array<Gesture>, shape: Array<number> | undefined): Array<Person> {
let id = 0; let id = 0;
const persons: Array<Person> = []; const persons: Array<Person> = [];
for (const face of faces) { // person is defined primarily by face and then we append other objects as found for (const face of faces) { // person is defined primarily by face and then we append other objects as found
const person: Person = { id: id++, face, body: null, hands: { left: null, right: null }, gestures: [], box: [0, 0, 0, 0] }; const person: Person = {
id: id++,
face,
body: null,
hands: { left: null, right: null },
gestures: {
face: [],
iris: [],
body: [],
hand: [],
},
box: [0, 0, 0, 0],
};
for (const body of bodies) { for (const body of bodies) {
if (face.box[0] > body.box[0] // x within body if (face.box[0] > body.box[0] // x within body
&& face.box[0] < body.box[0] + body.box[2] && face.box[0] < body.box[0] + body.box[2]
@ -34,11 +45,11 @@ export function join(faces: Array<Face>, bodies: Array<Body>, hands: Array<Hand>
} }
} }
for (const gesture of gestures) { // append all gestures according to ids for (const gesture of gestures) { // append all gestures according to ids
if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures?.push(gesture); if (gesture['face'] !== undefined && gesture['face'] === face.id) person.gestures.face.push(gesture.gesture as FaceGesture);
else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures?.push(gesture); else if (gesture['iris'] !== undefined && gesture['iris'] === face.id) person.gestures.iris.push(gesture.gesture as IrisGesture);
else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures?.push(gesture); else if (gesture['body'] !== undefined && gesture['body'] === person.body?.id) person.gestures.body.push(gesture.gesture as BodyGesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures?.push(gesture); else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.left?.id) person.gestures.hand.push(gesture.gesture as HandGesture);
else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures?.push(gesture); else if (gesture['hand'] !== undefined && gesture['hand'] === person.hands?.right?.id) person.gestures.hand.push(gesture.gesture as HandGesture);
} }
// create new overarching box from all boxes beloning to person // create new overarching box from all boxes beloning to person

267
src/result.ts
View File

@ -4,6 +4,16 @@
import { Tensor } from './tfjs/types'; import { Tensor } from './tfjs/types';
/**
* @typedef Gender
*/
export type Gender = 'male' | 'female';
/**
* @typedef Emotion
*/
export type Emotion = 'angry' | 'disgust' | 'fear' | 'happy' | 'sad' | 'surprise' | 'neutral';
/** Face results /** Face results
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
* Some values may be null if specific model is not enabled * Some values may be null if specific model is not enabled
@ -32,28 +42,28 @@ import { Tensor } from './tfjs/types';
* - tensor: face tensor as Tensor object which contains detected face * - tensor: face tensor as Tensor object which contains detected face
*/ */
export interface Face { export interface Face {
id: number id: number;
score: number, score: number;
boxScore: number, boxScore: number;
faceScore: number, faceScore: number;
box: [number, number, number, number], box: [number, number, number, number];
boxRaw: [number, number, number, number], boxRaw: [number, number, number, number];
mesh: Array<[number, number, number]> mesh: Array<[number, number, number]>;
meshRaw: Array<[number, number, number]> meshRaw: Array<[number, number, number]>;
annotations: Record<string, Array<[number, number, number]>>, annotations: Record<string, Array<[number, number, number]>>;
age?: number, age?: number;
gender?: string, gender?: Gender;
genderScore?: number, genderScore?: number;
emotion?: Array<{ score: number, emotion: string }>, emotion?: Array<{ score: number; emotion: Emotion }>;
embedding?: Array<number>, embedding?: Array<number>;
iris?: number, iris?: number;
rotation?: { rotation?: {
angle: { roll: number, yaw: number, pitch: number }, angle: { roll: number; yaw: number; pitch: number };
matrix: [number, number, number, number, number, number, number, number, number], matrix: [number, number, number, number, number, number, number, number, number];
gaze: { bearing: number, strength: number }, gaze: { bearing: number; strength: number };
} };
image?: Tensor; image?: Tensor;
tensor: Tensor, tensor: Tensor;
} }
/** Body results /** Body results
@ -70,17 +80,17 @@ export interface Face {
* - presence: body part presence value * - presence: body part presence value
*/ */
export interface Body { export interface Body {
id: number, id: number;
score: number, score: number;
box: [number, number, number, number], box: [number, number, number, number];
boxRaw: [number, number, number, number], boxRaw: [number, number, number, number];
keypoints: Array<{ keypoints: Array<{
part: string, part: string;
position: [number, number, number?], position: [number, number, number?];
positionRaw: [number, number, number?], positionRaw: [number, number, number?];
score: number, score: number;
presence?: number, presence?: number;
}> }>;
} }
/** Hand results /** Hand results
@ -94,69 +104,144 @@ export interface Body {
* - annotations: annotated landmarks for each hand part * - annotations: annotated landmarks for each hand part
*/ */
export interface Hand { export interface Hand {
id: number, id: number;
score: number, score: number;
box: [number, number, number, number], box: [number, number, number, number];
boxRaw: [number, number, number, number], boxRaw: [number, number, number, number];
keypoints: Array<[number, number, number]>, keypoints: Array<[number, number, number]>;
annotations: Record<string, Array<[number, number, number]>>, annotations: Record<string, Array<[number, number, number]>>;
} }
/** Object results /** Object results
*
* Array of individual results with one object per detected gesture
* Each result has:
* - id: object id number
* - score as value
* - label as detected class name
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
* - center: optional center point as array of [x, y], normalized to image resolution
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
*/
export interface Item {
id: number,
score: number,
class: number,
label: string,
box: [number, number, number, number],
boxRaw: [number, number, number, number],
}
/** Gesture results
* @typedef Gesture Type
* *
* Array of individual results with one object per detected gesture * Array of individual results with one object per detected gesture
* Each result has: * Each result has:
* - part: part name and number where gesture was detected: face, iris, body, hand * - id: object id number
* - gesture: gesture detected * - score as value
* - label as detected class name
* - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
* - center: optional center point as array of [x, y], normalized to image resolution
* - centerRaw: optional center point as array of [x, y], normalized to range 0..1
*/ */
export interface Item {
id: number;
score: number;
class: number;
label: string;
box: [number, number, number, number];
boxRaw: [number, number, number, number];
}
/**
* @typedef FaceGesture
*/
export type FaceGesture =
| `facing ${'left' | 'center' | 'right'}`
| `blink ${'left' | 'right'} eye`
| `mouth ${number}% open`
| `head ${'up' | 'down'}`;
/**
* @typedef IrisGesture
*/
export type IrisGesture =
| 'facing center'
| `looking ${'left' | 'right' | 'up' | 'down'}`
| 'looking center';
/**
* @typedef BodyGesture
*/
export type BodyGesture =
| `leaning ${'left' | 'right'}`
| `raise ${'left' | 'right'} hand`
| 'i give up';
/**
* @typedef FingerThumb
*/
export type FingerThumb = 'thumb';
/**
* @typedef FingerIndex
*/
export type FingerIndex = 'indexfinger';
/**
* @typedef FingerMiddle
*/
export type FingerMiddle = 'middlefinger';
/**
* @typedef FingerRing
*/
export type FingerRing = 'ringfinger';
/**
* @typedef FingerPinky
*/
export type FingerPinky = 'pinky';
/**
* @typedef FingerGesture
*/
export type FingerGesture = FingerThumb | FingerIndex | FingerMiddle | FingerRing | FingerPinky;
/**
* @typedef HandGestureType
*/
export type HandGestureType<T extends FingerGesture> = `${T} forward ${Exclude<FingerGesture, T>} up`
/**
* @typedef HandGesture
*/
export type HandGesture =
| HandGestureType<FingerThumb>
| HandGestureType<FingerIndex>
| HandGestureType<FingerMiddle>
| HandGestureType<FingerRing>
| HandGestureType<FingerPinky>
/** Gesture results
* @typedef Gesture Type
*
* Array of individual results with one object per detected gesture
* Each result has:
* - part: part name and number where gesture was detected: face, iris, body, hand
* - gesture: gesture detected
*/
export type Gesture = export type Gesture =
{ 'face': number, gesture: string } | { face: number; gesture: FaceGesture }
| { 'iris': number, gesture: string } | { iris: number; gesture: IrisGesture }
| { 'body': number, gesture: string } | { body: number; gesture: BodyGesture }
| { 'hand': number, gesture: string } | { hand: number; gesture: HandGesture };
/** Person getter /** Person getter
* @interface Person Interface * @interface Person Interface
* *
* Each result has: * Each result has:
* - id: person id * - id: person id
* - face: face object * - face: face object
* - body: body object * - body: body object
* - hands: array of hand objects * - hands: hands object
* - gestures: array of gestures * - gestures: gestures object
* - box: bounding box: x, y, width, height normalized to input image resolution * - box: bounding box: x, y, width, height normalized to input image resolution
* - boxRaw: bounding box: x, y, width, height normalized to 0..1 * - boxRaw: bounding box: x, y, width, height normalized to 0..1
*/ */
export interface Person { export interface Person {
id: number, id: number;
face: Face, face: Face;
body: Body | null, body: Body | null;
hands: { left: Hand | null, right: Hand | null }, hands: { left: Hand | null; right: Hand | null };
gestures: Array<Gesture>, gestures: {
box: [number, number, number, number], face: FaceGesture[];
boxRaw?: [number, number, number, number], iris: IrisGesture[];
body: BodyGesture[];
hand: HandGesture[];
};
box: [number, number, number, number];
boxRaw?: [number, number, number, number];
} }
/** /**
@ -166,21 +251,21 @@ export interface Person {
*/ */
export interface Result { export interface Result {
/** {@link Face}: detection & analysis results */ /** {@link Face}: detection & analysis results */
face: Array<Face>, face: Array<Face>;
/** {@link Body}: detection & analysis results */ /** {@link Body}: detection & analysis results */
body: Array<Body>, body: Array<Body>;
/** {@link Hand}: detection & analysis results */ /** {@link Hand}: detection & analysis results */
hand: Array<Hand>, hand: Array<Hand>;
/** {@link Gesture}: detection & analysis results */ /** {@link Gesture}: detection & analysis results */
gesture: Array<Gesture>, gesture: Array<Gesture>;
/** {@link Object}: detection & analysis results */ /** {@link Object}: detection & analysis results */
object: Array<Item> object: Array<Item>;
/** global performance object with timing values for each operation */ /** global performance object with timing values for each operation */
performance: Record<string, unknown>, performance: Record<string, unknown>;
/** optional processed canvas that can be used to draw input on screen */ /** optional processed canvas that can be used to draw input on screen */
canvas?: OffscreenCanvas | HTMLCanvasElement, canvas?: OffscreenCanvas | HTMLCanvasElement;
/** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */ /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
readonly timestamp: number, readonly timestamp: number;
/** getter property that returns unified persons object */ /** getter property that returns unified persons object */
persons: Array<Person>, persons: Array<Person>;
} }