ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add additional hand gestures

pull/356/head
Vladimir Mandic 2021-11-08 07:36:26 -05:00
parent 33d6e94787
commit 37f62f47fa
10 changed files with 103 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.build.json
View File

@ -126,7 +126,7 @@
"format": "iife",
"input": "src/human.ts",
"output": "dist/human.js",
"minify": false,
"minify": true,
"globalName": "Human",
"external": ["fs", "os", "buffer", "util"]
},

4
CHANGELOG.md
View File

@ -9,8 +9,10 @@
## Changelog
### **HEAD -> main** 2021/11/07 mandic00@live.com
### **2.5.1** 2021/11/08 mandic00@live.com
- new human.compare api
- added links to release notes
### **origin/main** 2021/11/06 mandic00@live.com

31
TODO.md
View File

@ -38,34 +38,3 @@ MoveNet MultiPose model does not work with WASM backend due to missing F32 broad
<https://github.com/tensorflow/tfjs/issues/5516>
<br><hr><br>
### Pending release
New:
- New frame change detection algorithm used for [cache determination](https://vladmandic.github.io/human/typedoc/interfaces/Config.html#cacheSensitivity)
based on temporal input difference
- New built-in Tensorflow profiling [human.profile](https://vladmandic.github.io/human/typedoc/classes/Human.html#profile)
- New optional input histogram equalization [config.filter.equalization](https://vladmandic.github.io/human/)
auto-level input for optimal brightness/contrast
- New event-baseed interface [human.events](https://vladmandic.github.io/human/typedoc/classes/Human.html#events)
- New configuration validation [human.validate](https://vladmandic.github.io/human/typedoc/classes/Human.html#validate)
- New input compare function [human.compare](https://vladmandic.github.io/human/typedoc/classes/Human.html#compare)
this function is internally used by `human` to determine frame changes and cache validation
- New [custom built TFJS](https://github.com/vladmandic/tfjs) for bundled version
result is a pure module with reduced bundle size and include built-in support for all backends
note: **nobundle** and **node** versions link to standard `@tensorflow` packages
Changed:
- [Default configuration values](https://github.com/vladmandic/human/blob/main/src/config.ts#L262) have been tuned for precision and performance
- Supports all built-in modules on all backends
via custom implementation of missing kernel ops
- Performance and precision improvements
- **face**, **hand**
- **gestures** modules
- **face matching**
- Fix **ReactJS** compatibility
- Better precision using **WASM**
Previous issues due to math low-precision in WASM implementation
- Full **TS** type definitions for all modules and imports
- Focus on simplified demo
<https://vladmandic.github.io/human/demo/typescript/>

15
src/face/facemesh.ts
View File

@ -68,12 +68,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
annotations: {},
};
if (config.face.detector?.rotation && config.face.mesh?.enabled && env.kernels.includes('rotatewithoffset')) {
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(box, input, inputSize);
} else {
rotationMatrix = util.fixedRotationMatrix;
face.tensor = util.cutBoxFromImageAndResize(box, input, config.face.mesh?.enabled ? [inputSize, inputSize] : [blazeface.size(), blazeface.size()]);
}
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(false && config.face.detector?.rotation, box, input, inputSize); // optional rotate based on detector data // disabled
if (config?.filter?.equalization) {
const equilized = await histogramEqualization(face.tensor as Tensor);
tf.dispose(face.tensor);
@ -112,14 +107,8 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
face.boxRaw = util.getRawBox(box, input);
face.score = face.faceScore;
newCache.push(box);
// other modules prefer different crop for a face so we dispose it and do it again
/*
tf.dispose(face.tensor);
face.tensor = config.face.detector?.rotation && config.face.mesh?.enabled && env.kernels.includes('rotatewithoffset')
? face.tensor = util.correctFaceRotation(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, inputSize)[2]
: face.tensor = util.cutBoxFromImageAndResize(util.enlargeBox(box, Math.sqrt(enlargeFact)), input, [inputSize, inputSize]);
*/
[angle, rotationMatrix, face.tensor] = util.correctFaceRotation(config.face.detector?.rotation, box, input, inputSize); // optional rotate once more based on mesh data
}
}
faces.push(face);

36
src/face/facemeshutil.ts
View File

@ -6,6 +6,7 @@
import * as tf from '../../dist/tfjs.esm.js';
import * as coords from './facemeshcoords';
import type { Box, Point } from '../result';
import { env } from '../util/env';
export const createBox = (startEndTensor) => ({ startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]), endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]) });
@ -155,21 +156,28 @@ export function transformRawCoords(coordsRaw, box, angle, rotationMatrix, inputS
]));
}
export function correctFaceRotation(box, input, inputSize) {
const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count) ? coords.meshLandmarks.symmetryLine : coords.blazeFaceLandmarks.symmetryLine;
const angle: number = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]);
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
let rotationMatrix;
let face;
if (largeAngle) {
const faceCenter: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const faceCenterNormalized: Point = [faceCenter[0] / input.shape[2], faceCenter[1] / input.shape[1]];
const rotated = tf.image.rotateWithOffset(input, angle, 0, faceCenterNormalized); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = buildRotationMatrix(-angle, faceCenter);
face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]);
tf.dispose(rotated);
export function correctFaceRotation(rotate, box, input, inputSize) {
const symmetryLine = (box.landmarks.length >= coords.meshLandmarks.count)
? coords.meshLandmarks.symmetryLine
: coords.blazeFaceLandmarks.symmetryLine;
let angle = 0; // default
let rotationMatrix = fixedRotationMatrix; // default
let face; // default
if (rotate && env.kernels.includes('rotatewithoffset')) {
angle = computeRotation(box.landmarks[symmetryLine[0]], box.landmarks[symmetryLine[1]]);
const largeAngle = angle && (angle !== 0) && (Math.abs(angle) > 0.2);
if (largeAngle) {
const center: Point = getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint });
const centerRaw: Point = [center[0] / input.shape[2], center[1] / input.shape[1]];
const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw); // rotateWithOffset is not defined for tfjs-node
rotationMatrix = buildRotationMatrix(-angle, center);
face = cutBoxFromImageAndResize(box, rotated, [inputSize, inputSize]);
tf.dispose(rotated);
} else {
face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]);
}
} else {
rotationMatrix = fixedRotationMatrix;
face = cutBoxFromImageAndResize(box, input, [inputSize, inputSize]);
}
return [angle, rotationMatrix, face];

6
src/hand/fingerdef.ts
View File

@ -66,17 +66,17 @@ export class FingerGesture {
this.weightsRelative = [1.0, 1.0, 1.0, 1.0, 1.0];
}
addCurl(finger, curl, confidence) {
curl(finger, curl, confidence) {
if (typeof this.curls[finger] === 'undefined') this.curls[finger] = [];
this.curls[finger].push([curl, confidence]);
}
addDirection(finger, position, confidence) {
direction(finger, position, confidence) {
if (!this.directions[finger]) this.directions[finger] = [];
this.directions[finger].push([position, confidence]);
}
setWeight(finger, weight) {
weight(finger, weight) {
this.weights[finger] = weight;
// recalculate relative weights
const total = this.weights.reduce((a, b) => a + b, 0);

91
src/hand/fingergesture.ts
View File

@ -5,39 +5,74 @@
import { Finger, FingerCurl, FingerDirection, FingerGesture } from './fingerdef';
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { thumb, index, middle, ring, pinky } = Finger;
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { none, half, full } = FingerCurl;
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const { verticalUp, verticalDown, horizontalLeft, horizontalRight, diagonalUpRight, diagonalUpLeft, diagonalDownRight, diagonalDownLeft } = FingerDirection;
// describe thumbs up gesture 👍
const ThumbsUp = new FingerGesture('thumbs up');
ThumbsUp.addCurl(Finger.thumb, FingerCurl.none, 1.0);
ThumbsUp.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 0.25);
ThumbsUp.addDirection(Finger.thumb, FingerDirection.diagonalUpRight, 0.25);
ThumbsUp.curl(thumb, none, 1.0);
ThumbsUp.direction(thumb, verticalUp, 1.0);
ThumbsUp.direction(thumb, diagonalUpLeft, 0.25);
ThumbsUp.direction(thumb, diagonalUpRight, 0.25);
for (const finger of [Finger.index, Finger.middle, Finger.ring, Finger.pinky]) {
ThumbsUp.addCurl(finger, FingerCurl.full, 1.0);
ThumbsUp.addDirection(finger, FingerDirection.horizontalLeft, 1.0);
ThumbsUp.addDirection(finger, FingerDirection.horizontalRight, 1.0);
ThumbsUp.curl(finger, full, 1.0);
ThumbsUp.direction(finger, horizontalLeft, 1.0);
ThumbsUp.direction(finger, horizontalRight, 1.0);
}
// describe Victory gesture ✌️
const Victory = new FingerGesture('victory');
Victory.addCurl(Finger.thumb, FingerCurl.half, 0.5);
Victory.addCurl(Finger.thumb, FingerCurl.none, 0.5);
Victory.addDirection(Finger.thumb, FingerDirection.verticalUp, 1.0);
Victory.addDirection(Finger.thumb, FingerDirection.diagonalUpLeft, 1.0);
Victory.addCurl(Finger.index, FingerCurl.none, 1.0);
Victory.addDirection(Finger.index, FingerDirection.verticalUp, 0.75);
Victory.addDirection(Finger.index, FingerDirection.diagonalUpLeft, 1.0);
Victory.addCurl(Finger.middle, FingerCurl.none, 1.0);
Victory.addDirection(Finger.middle, FingerDirection.verticalUp, 1.0);
Victory.addDirection(Finger.middle, FingerDirection.diagonalUpLeft, 0.75);
Victory.addCurl(Finger.ring, FingerCurl.full, 1.0);
Victory.addDirection(Finger.ring, FingerDirection.verticalUp, 0.2);
Victory.addDirection(Finger.ring, FingerDirection.diagonalUpLeft, 1.0);
Victory.addDirection(Finger.ring, FingerDirection.horizontalLeft, 0.2);
Victory.addCurl(Finger.pinky, FingerCurl.full, 1.0);
Victory.addDirection(Finger.pinky, FingerDirection.verticalUp, 0.2);
Victory.addDirection(Finger.pinky, FingerDirection.diagonalUpLeft, 1.0);
Victory.addDirection(Finger.pinky, FingerDirection.horizontalLeft, 0.2);
Victory.setWeight(Finger.index, 2);
Victory.setWeight(Finger.middle, 2);
Victory.curl(thumb, half, 0.5);
Victory.curl(thumb, none, 0.5);
Victory.direction(thumb, verticalUp, 1.0);
Victory.direction(thumb, diagonalUpLeft, 1.0);
Victory.curl(index, none, 1.0);
Victory.direction(index, verticalUp, 0.75);
Victory.direction(index, diagonalUpLeft, 1.0);
Victory.curl(middle, none, 1.0);
Victory.direction(middle, verticalUp, 1.0);
Victory.direction(middle, diagonalUpLeft, 0.75);
Victory.curl(ring, full, 1.0);
Victory.direction(ring, verticalUp, 0.2);
Victory.direction(ring, diagonalUpLeft, 1.0);
Victory.direction(ring, horizontalLeft, 0.2);
Victory.curl(pinky, full, 1.0);
Victory.direction(pinky, verticalUp, 0.2);
Victory.direction(pinky, diagonalUpLeft, 1.0);
Victory.direction(pinky, horizontalLeft, 0.2);
Victory.weight(index, 2);
Victory.weight(middle, 2);
export default [ThumbsUp, Victory];
// describe Point gesture ✌️
const Point = new FingerGesture('point');
Point.curl(thumb, full, 1.0);
Point.curl(index, none, 0.5);
Point.curl(middle, full, 0.5);
Point.curl(ring, full, 0.5);
Point.curl(pinky, full, 0.5);
Point.weight(index, 2);
Point.weight(middle, 2);
// describe Point gesture ✌️
const MiddleFinger = new FingerGesture('middle finger');
MiddleFinger.curl(thumb, none, 1.0);
MiddleFinger.curl(index, full, 0.5);
MiddleFinger.curl(middle, full, 0.5);
MiddleFinger.curl(ring, full, 0.5);
MiddleFinger.curl(pinky, full, 0.5);
MiddleFinger.weight(index, 2);
MiddleFinger.weight(middle, 2);
// describe Open Palm gesture ✌️
const OpenPalm = new FingerGesture('open palm');
OpenPalm.curl(thumb, none, 0.75);
OpenPalm.curl(index, none, 0.75);
OpenPalm.curl(middle, none, 0.75);
OpenPalm.curl(ring, none, 0.75);
OpenPalm.curl(pinky, none, 0.75);
export default [ThumbsUp, Victory, Point, MiddleFinger, OpenPalm];

6
src/hand/handtrack.ts
View File

@ -24,7 +24,7 @@ const inputSize = [[0, 0], [0, 0]];
const classes = ['hand', 'fist', 'pinch', 'point', 'face', 'tip', 'pinchtip'];
const faceIndex = 4;
const boxExpandFact = 1.7;
const boxExpandFact = 1.6;
const maxDetectorResolution = 512;
const detectorExpandFact = 1.4;
@ -170,9 +170,7 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
outputSize[1] * (kpt[1] + h.boxRaw[1]),
(kpt[2] || 0),
]);
// hand.box = box.scale(h.box, 1 / detectorExpandFact); // scale box down for visual appeal
// hand.boxRaw = box.scale(h.boxRaw, 1 / detectorExpandFact); // scale box down for visual appeal
hand.landmarks = fingerPose.analyze(hand.keypoints) as HandResult['landmarks']; // calculate finger landmarks
hand.landmarks = fingerPose.analyze(hand.keypoints) as HandResult['landmarks']; // calculate finger gestures
for (const key of Object.keys(fingerMap)) { // map keypoints to per-finger annotations
hand.annotations[key] = fingerMap[key].map((index) => (hand.landmarks && hand.keypoints[index] ? hand.keypoints[index] : null));
}

12
test/test-main.js
View File

@ -153,9 +153,9 @@ async function verifyDetails(human) {
verify(res.face.length === 1, 'details face length', res.face.length);
for (const face of res.face) {
verify(face.score > 0.9 && face.boxScore > 0.9 && face.faceScore > 0.9, 'details face score', face.score, face.boxScore, face.faceScore);
verify(face.age > 29 && face.age < 30 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
verify(face.age > 23 && face.age < 24 && face.gender === 'female' && face.genderScore > 0.9 && face.iris > 70 && face.iris < 80, 'details face age/gender', face.age, face.gender, face.genderScore, face.iris);
verify(face.box.length === 4 && face.boxRaw.length === 4 && face.mesh.length === 478 && face.meshRaw.length === 478 && face.embedding.length === 1024, 'details face arrays', face.box.length, face.mesh.length, face.embedding.length);
verify(face.emotion.length === 3 && face.emotion[0].score > 0.5 && face.emotion[0].emotion === 'angry', 'details face emotion', face.emotion.length, face.emotion[0]);
verify(face.emotion.length === 3 && face.emotion[0].score > 0.45 && face.emotion[0].emotion === 'neutral', 'details face emotion', face.emotion.length, face.emotion[0]);
}
verify(res.body.length === 1, 'details body length', res.body.length);
for (const body of res.body) {
@ -166,7 +166,7 @@ async function verifyDetails(human) {
verify(hand.score > 0.5 && hand.boxScore > 0.5 && hand.fingerScore > 0.5 && hand.box.length === 4 && hand.boxRaw.length === 4 && hand.label === 'point', 'details hand', hand.boxScore, hand.fingerScore, hand.label);
verify(hand.keypoints.length === 21 && Object.keys(hand.landmarks).length === 5 && Object.keys(hand.annotations).length === 6, 'details hand arrays', hand.keypoints.length, Object.keys(hand.landmarks).length, Object.keys(hand.annotations).length);
}
verify(res.gesture.length === 5, 'details gesture length', res.gesture.length);
verify(res.gesture.length === 6, 'details gesture length', res.gesture.length);
verify(res.gesture[0].gesture === 'facing right', 'details gesture first', res.gesture[0]);
verify(res.object.length === 1, 'details object length', res.object.length);
for (const obj of res.object) {
@ -236,11 +236,11 @@ async function test(Human, inputConfig) {
else log('state', 'passed: warmup none result match');
config.warmup = 'face';
res = await testWarmup(human, 'default');
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length < 7) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
else log('state', 'passed: warmup face result match');
config.warmup = 'body';
res = await testWarmup(human, 'default');
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 5) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 1 || res?.gesture?.length !== 6) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
else log('state', 'passed: warmup body result match');
log('state', 'details:', {
face: { boxScore: res.face[0].boxScore, faceScore: res.face[0].faceScore, age: res.face[0].age, gender: res.face[0].gender, genderScore: res.face[0].genderScore },
@ -330,7 +330,7 @@ async function test(Human, inputConfig) {
config.body = { minConfidence: 0.0001 };
config.hand = { minConfidence: 0.0001 };
res = await testDetect(human, 'samples/in/ai-body.jpg', 'default');
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 7) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 2 || res?.gesture?.length !== 8) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length);
else log('state', 'passed: sensitive result match');
// test sensitive details face

2
wiki

@ -1 +1 @@
Subproject commit 55876f5dbb01d605c862ab2749a2bc11f86d5cba
Subproject commit 214e797620e397f16132ae38d6a9d6bd46f2c60d