From e84e421a047e9ddc766f2faa209aad0ad98635ed Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 29 Jul 2021 11:01:50 -0400 Subject: [PATCH] updated gesture types --- .eslintrc.json | 4 +++- CHANGELOG.md | 5 ++++- src/gesture/gesture.ts | 43 +++++++++++++++++++++++++++++++++++++----- src/result.ts | 9 +++++---- test/README.md | 8 +------- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 471672ff..9cad12ae 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -26,6 +26,7 @@ "rules": { "@typescript-eslint/ban-ts-comment": "off", "@typescript-eslint/explicit-module-boundary-types": "off", + "@typescript-eslint/no-shadow": "error", "@typescript-eslint/no-var-requires": "off", "camelcase": "off", "dot-notation": "off", @@ -44,6 +45,7 @@ "no-bitwise": "off", "no-case-declarations":"off", "no-continue": "off", + "no-lonely-if": "off", "no-loop-func": "off", "no-mixed-operators": "off", "no-param-reassign":"off", @@ -52,12 +54,12 @@ "no-restricted-globals": "off", "no-restricted-syntax": "off", "no-return-assign": "off", + "no-shadow": "off", "no-underscore-dangle": "off", "node/no-missing-import": ["error", { "tryExtensions": [".js", ".json", ".ts"] }], "node/no-unpublished-import": "off", "node/no-unpublished-require": "off", "node/no-unsupported-features/es-syntax": "off", - "no-lonely-if": "off", "node/shebang": "off", "object-curly-newline": "off", "prefer-destructuring": "off", diff --git a/CHANGELOG.md b/CHANGELOG.md index 81cbe6c9..19d931e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,10 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/07/19 marcogodoy@untitled.cl +### **HEAD -> main** 2021/07/29 mandic00@live.com + + +### **origin/main** 2021/07/19 marcogodoy@untitled.cl - add note on manually disping tensor - modularize model loading diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index e49e39d3..ee63ea77 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -4,9 +4,41 @@ import { Gesture } from '../result'; +/** + * @typedef FaceGesture + */ +export type FaceGesture = + `facing ${'left' | 'center' | 'right'}` + | `blink ${'left' | 'right'} eye` + | `mouth ${number}% open` + | `head ${'up' | 'down'}`; + +/** + * @typedef IrisGesture + */ +export type IrisGesture = + 'facing center' + | `looking ${'left' | 'right' | 'up' | 'down'}` + | 'looking center'; + +/** + * @typedef BodyGesture + */ +export type BodyGesture = + `leaning ${'left' | 'right'}` + | `raise ${'left' | 'right'} hand` + | 'i give up'; + +/** + * @typedef BodyGesture + */ +export type HandGesture = + `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} forward` + | `${'thumb' | 'index finger' | 'middle finger' | 'ring finger' | 'pinky'} up`; + export const body = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ body: number, gesture: string }> = []; + const gestures: Array<{ body: number, gesture: BodyGesture }> = []; for (let i = 0; i < res.length; i++) { // raising hands const leftWrist = res[i].keypoints.find((a) => (a.part === 'leftWrist')); @@ -26,7 +58,7 @@ export const body = (res): Gesture[] => { export const face = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ face: number, gesture: string }> = []; + const gestures: Array<{ face: number, gesture: FaceGesture }> = []; for (let i = 0; i < res.length; i++) { if (res[i].mesh && res[i].mesh.length > 0) { const eyeFacing = res[i].mesh[33][2] - res[i].mesh[263][2]; @@ -47,7 +79,7 @@ export const face = (res): Gesture[] => { export const iris = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ iris: number, gesture: string }> = []; + const gestures: Array<{ iris: number, gesture: IrisGesture }> = []; for (let i = 0; i < res.length; i++) { if (!res[i].annotations || !res[i].annotations.leftEyeIris || !res[i].annotations.rightEyeIris) continue; const sizeXLeft = res[i].annotations.leftEyeIris[3][0] - res[i].annotations.leftEyeIris[1][0]; @@ -85,7 +117,7 @@ export const iris = (res): Gesture[] => { export const hand = (res): Gesture[] => { if (!res) return []; - const gestures: Array<{ hand: number, gesture: string }> = []; + const gestures: Array<{ hand: number, gesture: HandGesture }> = []; for (let i = 0; i < res.length; i++) { const fingers: Array<{ name: string, position: number }> = []; for (const [finger, pos] of Object.entries(res[i]['annotations'])) { @@ -93,8 +125,9 @@ export const hand = (res): Gesture[] => { } if (fingers && fingers.length > 0) { const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); + gestures.push({ hand: i, gesture: `${closest.name} forward` as HandGesture }); const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a)); - gestures.push({ hand: i, gesture: `${closest.name} forward ${highest.name} up` }); + gestures.push({ hand: i, gesture: `${highest.name} up` as HandGesture }); } } return gestures; diff --git a/src/result.ts b/src/result.ts index 8dbf3084..11babac2 100644 --- a/src/result.ts +++ b/src/result.ts @@ -3,6 +3,7 @@ */ import { Tensor } from './tfjs/types'; +import { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; /** Face results * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models @@ -132,10 +133,10 @@ export interface Item { * - gesture: gesture detected */ export type Gesture = - { 'face': number, gesture: string } - | { 'iris': number, gesture: string } - | { 'body': number, gesture: string } - | { 'hand': number, gesture: string } + { 'face': number, gesture: FaceGesture } + | { 'iris': number, gesture: IrisGesture } + | { 'body': number, gesture: BodyGesture } + | { 'hand': number, gesture: HandGesture } /** Person getter * @interface Person Interface diff --git a/test/README.md b/test/README.md index bfa18c60..4f42eb47 100644 --- a/test/README.md +++ b/test/README.md @@ -10,13 +10,7 @@ Not required for normal funcioning of library - Face rotation is disabled for `NodeJS` platform: `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'` - -### NodeJS with GPU acceleation using CUDA - -- Image filters are disabled due to lack of Canvas and WeBGL access -- Face rotation is disabled for `NodeJS` platform: - `Kernel 'RotateWithOffset' not registered for backend 'tensorflow'` - + Work has recently been completed and will likely be included in TFJS 3.9.0 ### NodeJS using WASM