From 37f62f47fa336cd7b9a2d59c916c594415fb744f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 8 Nov 2021 07:36:26 -0500 Subject: [PATCH] add additional hand gestures --- .build.json | 2 +- CHANGELOG.md | 4 +- TODO.md | 31 ------------- src/face/facemesh.ts | 15 +------ src/face/facemeshutil.ts | 36 ++++++++++------ src/hand/fingerdef.ts | 6 +-- src/hand/fingergesture.ts | 91 +++++++++++++++++++++++++++------------ src/hand/handtrack.ts | 6 +-- test/test-main.js | 12 +++--- wiki | 2 +- 10 files changed, 103 insertions(+), 102 deletions(-) diff --git a/.build.json b/.build.json index 927f3c8d..344ce9b0 100644 --- a/.build.json +++ b/.build.json @@ -126,7 +126,7 @@ "format": "iife", "input": "src/human.ts", "output": "dist/human.js", - "minify": false, + "minify": true, "globalName": "Human", "external": ["fs", "os", "buffer", "util"] }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 728a59b2..b16e62ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ ## Changelog -### **HEAD -> main** 2021/11/07 mandic00@live.com +### **2.5.1** 2021/11/08 mandic00@live.com +- new human.compare api +- added links to release notes ### **origin/main** 2021/11/06 mandic00@live.com diff --git a/TODO.md b/TODO.md index 2ae3ce77..41260b76 100644 --- a/TODO.md +++ b/TODO.md @@ -38,34 +38,3 @@ MoveNet MultiPose model does not work with WASM backend due to missing F32 broad

- -### Pending release - -New: -- New frame change detection algorithm used for [cache determination](https://vladmandic.github.io/human/typedoc/interfaces/Config.html#cacheSensitivity) - based on temporal input difference -- New built-in Tensorflow profiling [human.profile](https://vladmandic.github.io/human/typedoc/classes/Human.html#profile) -- New optional input histogram equalization [config.filter.equalization](https://vladmandic.github.io/human/) - auto-level input for optimal brightness/contrast -- New event-baseed interface [human.events](https://vladmandic.github.io/human/typedoc/classes/Human.html#events) -- New configuration validation [human.validate](https://vladmandic.github.io/human/typedoc/classes/Human.html#validate) -- New input compare function [human.compare](https://vladmandic.github.io/human/typedoc/classes/Human.html#compare) - this function is internally used by `human` to determine frame changes and cache validation -- New [custom built TFJS](https://github.com/vladmandic/tfjs) for bundled version - result is a pure module with reduced bundle size and include built-in support for all backends - note: **nobundle** and **node** versions link to standard `@tensorflow` packages - -Changed: -- [Default configuration values](https://github.com/vladmandic/human/blob/main/src/config.ts#L262) have been tuned for precision and performance -- Supports all built-in modules on all backends - via custom implementation of missing kernel ops -- Performance and precision improvements - - **face**, **hand** - - **gestures** modules - - **face matching** -- Fix **ReactJS** compatibility -- Better precision using **WASM** - Previous issues due to math low-precision in WASM implementation -- Full **TS** type definitions for all modules and imports -- Focus on simplified demo - diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index ff59794f..bf3f96d7 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -68,12 +68,7 @@ export async function predict(input: Tensor, config: Config): Promise ({ startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]) }); @@ -155,21 +156,28 @@ export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputS ])); } -export function correctFaceRotation(box, input, inputSize) { - const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count) ? coords.meshLandmarks.symmetryLine : coords.blazeFaceLandmarks.symmetryLine; - const angle: number = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]); - const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); - let rotationMatrix; - let face; - if (largeAngle) { - const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); - const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]]; - const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node - rotationMatrix = buildRotationMatrix(-angle, faceCenter); - face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]); - tf.dispose(rotated); +export function correctFaceRotation(rotate, box, input, inputSize) { + const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count) + ? coords.meshLandmarks.symmetryLine + : coords.blazeFaceLandmarks.symmetryLine; + let angle = 0; // default + let rotationMatrix = fixedRotationMatrix; // default + let face; // default + + if (rotate && env.kernels.includes('rotatewithoffset')) { + angle = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]); + const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2); + if (largeAngle) { + const center: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }); + const centerRaw: Point = [center[0] / input.shape[2], center[1] / input.shape[1]]; + const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw); // rotateWithOffset is not defined for tfjs-node + rotationMatrix = buildRotationMatrix(-angle, center); + face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]); + tf.dispose(rotated); + } else { + face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]); + } } else { - rotationMatrix = fixedRotationMatrix; face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]); } return [angle, rotationMatrix, face]; diff --git a/src/hand/fingerdef.ts b/src/hand/fingerdef.ts index 2d348299..a0c1b41e 100644 --- a/src/hand/fingerdef.ts +++ b/src/hand/fingerdef.ts @@ -66,17 +66,17 @@ export class FingerGesture { this.weightsRelative = [1.0, 1.0, 1.0, 1.0, 1.0]; } - addCurl(finger, curl, confidence) { + curl(finger, curl, confidence) { if (typeof this.curls[finger] === 'undefined') this.curls[finger] = []; this.curls[finger].push([curl, confidence]); } - addDirection(finger, position, confidence) { + direction(finger, position, confidence) { if (!this.directions[finger]) this.directions[finger] = []; this.directions[finger].push([position, confidence]); } - setWeight(finger, weight) { + weight(finger, weight) { this.weights[finger] = weight; // recalculate relative weights const total = this.weights.reduce((a, b) => a + b, 0); diff --git a/src/hand/fingergesture.ts b/src/hand/fingergesture.ts index f25d1047..042a7a74 100644 --- a/src/hand/fingergesture.ts +++ b/src/hand/fingergesture.ts @@ -5,39 +5,74 @@ import { Finger, FingerCurl, FingerDirection, FingerGesture } from './fingerdef'; +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +const { thumb, index, middle, ring, pinky } = Finger; +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +const { none, half, full } = FingerCurl; +// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars +const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection; + // describe thumbs up gesture 👍 const ThumbsUp = new FingerGesture('thumbs up'); -ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1.0); -ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0); -ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25); -ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25); +ThumbsUp.curl(thumb, none, 1.0); +ThumbsUp.direction(thumb, verticalUp, 1.0); +ThumbsUp.direction(thumb, diagonalUpLeft, 0.25); +ThumbsUp.direction(thumb, diagonalUpRight, 0.25); for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) { - ThumbsUp.addCurl(finger, FingerCurl.full, 1.0); - ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1.0); - ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1.0); + ThumbsUp.curl(finger, full, 1.0); + ThumbsUp.direction(finger, horizontalLeft, 1.0); + ThumbsUp.direction(finger, horizontalRight, 1.0); } // describe Victory gesture ✌️ const Victory = new FingerGesture('victory'); -Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5); -Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5); -Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0); -Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1.0); -Victory.addCurl(Finger.index, FingerCurl.none, 1.0); -Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75); -Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1.0); -Victory.addCurl(Finger.middle, FingerCurl.none, 1.0); -Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1.0); -Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75); -Victory.addCurl(Finger.ring, FingerCurl.full, 1.0); -Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2); -Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1.0); -Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2); -Victory.addCurl(Finger.pinky, FingerCurl.full, 1.0); -Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2); -Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1.0); -Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2); -Victory.setWeight(Finger.index, 2); -Victory.setWeight(Finger.middle, 2); +Victory.curl(thumb, half, 0.5); +Victory.curl(thumb, none, 0.5); +Victory.direction(thumb, verticalUp, 1.0); +Victory.direction(thumb, diagonalUpLeft, 1.0); +Victory.curl(index, none, 1.0); +Victory.direction(index, verticalUp, 0.75); +Victory.direction(index, diagonalUpLeft, 1.0); +Victory.curl(middle, none, 1.0); +Victory.direction(middle, verticalUp, 1.0); +Victory.direction(middle, diagonalUpLeft, 0.75); +Victory.curl(ring, full, 1.0); +Victory.direction(ring, verticalUp, 0.2); +Victory.direction(ring, diagonalUpLeft, 1.0); +Victory.direction(ring, horizontalLeft, 0.2); +Victory.curl(pinky, full, 1.0); +Victory.direction(pinky, verticalUp, 0.2); +Victory.direction(pinky, diagonalUpLeft, 1.0); +Victory.direction(pinky, horizontalLeft, 0.2); +Victory.weight(index, 2); +Victory.weight(middle, 2); -export default [ThumbsUp, Victory]; +// describe Point gesture ✌️ +const Point = new FingerGesture('point'); +Point.curl(thumb, full, 1.0); +Point.curl(index, none, 0.5); +Point.curl(middle, full, 0.5); +Point.curl(ring, full, 0.5); +Point.curl(pinky, full, 0.5); +Point.weight(index, 2); +Point.weight(middle, 2); + +// describe Point gesture ✌️ +const MiddleFinger = new FingerGesture('middle finger'); +MiddleFinger.curl(thumb, none, 1.0); +MiddleFinger.curl(index, full, 0.5); +MiddleFinger.curl(middle, full, 0.5); +MiddleFinger.curl(ring, full, 0.5); +MiddleFinger.curl(pinky, full, 0.5); +MiddleFinger.weight(index, 2); +MiddleFinger.weight(middle, 2); + +// describe Open Palm gesture ✌️ +const OpenPalm = new FingerGesture('open palm'); +OpenPalm.curl(thumb, none, 0.75); +OpenPalm.curl(index, none, 0.75); +OpenPalm.curl(middle, none, 0.75); +OpenPalm.curl(ring, none, 0.75); +OpenPalm.curl(pinky, none, 0.75); + +export default [ThumbsUp, Victory, Point, MiddleFinger, OpenPalm]; diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index f3ee00db..79727a1b 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -24,7 +24,7 @@ const inputSize = [[0, 0], [0, 0]]; const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip']; const faceIndex = 4; -const boxExpandFact = 1.7; +const boxExpandFact = 1.6; const maxDetectorResolution = 512; const detectorExpandFact = 1.4; @@ -170,9 +170,7 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config) outputSize[1] * (kpt[1] + h.boxRaw[1]), (kpt[2] || 0), ]); - // hand.box = box.scale(h.box, 1 / detectorExpandFact); // scale box down for visual appeal - // hand.boxRaw = box.scale(h.boxRaw, 1 / detectorExpandFact); // scale box down for visual appeal - hand.landmarks = fingerPose.analyze(hand.keypoints) as HandResult['landmarks']; // calculate finger landmarks + hand.landmarks = fingerPose.analyze(hand.keypoints) as HandResult['landmarks']; // calculate finger gestures for (const key of Object.keys(fingerMap)) { // map keypoints to per-finger annotations hand.annotations[key] = fingerMap[key].map((index) => (hand.landmarks && hand.keypoints[index] ? hand.keypoints[index] : null)); } diff --git a/test/test-main.js b/test/test-main.js index b440cbb5..e9aadf69 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -153,9 +153,9 @@ async function verifyDetails(human) { verify(res.face.length === 1, 'details face length', res.face.length); for (const face of res.face) { verify(face.score > 0.9 && face.boxScore > 0.9 && face.faceScore > 0.9, 'details face score', face.score, face.boxScore, face.faceScore); - verify(face.age > 29 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris); + verify(face.age > 23 && face.age < 24 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris); verify(face.box.length === 4 && face.boxRaw.length === 4 && face.mesh.length === 478 && face.meshRaw.length === 478 && face.embedding.length === 1024, 'details face arrays', face.box.length, face.mesh.length, face.embedding.length); - verify(face.emotion.length === 3 && face.emotion[0].score > 0.5 && face.emotion[0].emotion === 'angry', 'details face emotion', face.emotion.length, face.emotion[0]); + verify(face.emotion.length === 3 && face.emotion[0].score > 0.45 && face.emotion[0].emotion === 'neutral', 'details face emotion', face.emotion.length, face.emotion[0]); } verify(res.body.length === 1, 'details body length', res.body.length); for (const body of res.body) { @@ -166,7 +166,7 @@ async function verifyDetails(human) { verify(hand.score > 0.5 && hand.boxScore > 0.5 && hand.fingerScore > 0.5 && hand.box.length === 4 && hand.boxRaw.length === 4 && hand.label === 'point', 'details hand', hand.boxScore, hand.fingerScore, hand.label); verify(hand.keypoints.length === 21 && Object.keys(hand.landmarks).length === 5 && Object.keys(hand.annotations).length === 6, 'details hand arrays', hand.keypoints.length, Object.keys(hand.landmarks).length, Object.keys(hand.annotations).length); } - verify(res.gesture.length === 5, 'details gesture length', res.gesture.length); + verify(res.gesture.length === 6, 'details gesture length', res.gesture.length); verify(res.gesture[0].gesture === 'facing right', 'details gesture first', res.gesture[0]); verify(res.object.length === 1, 'details object length', res.object.length); for (const obj of res.object) { @@ -236,11 +236,11 @@ async function test(Human, inputConfig) { else log('state', 'passed: warmup none result match'); config.warmup = 'face'; res = await testWarmup(human, 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length < 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: warmup face result match'); config.warmup = 'body'; res = await testWarmup(human, 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 5) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 6) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: warmup body result match'); log('state', 'details:', { face: { boxScore: res.face[0].boxScore, faceScore: res.face[0].faceScore, age: res.face[0].age, gender: res.face[0].gender, genderScore: res.face[0].genderScore }, @@ -330,7 +330,7 @@ async function test(Human, inputConfig) { config.body = { minConfidence: 0.0001 }; config.hand = { minConfidence: 0.0001 }; res = await testDetect(human, 'samples/in/ai-body.jpg', 'default'); - if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 7) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); else log('state', 'passed: sensitive result match'); // test sensitive details face diff --git a/wiki b/wiki index 55876f5d..214e7976 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 55876f5dbb01d605c862ab2749a2bc11f86d5cba +Subproject commit 214e797620e397f16132ae38d6a9d6bd46f2c60d