mirror of https://github.com/vladmandic/human
conditional hand rotation
Vladimir Mandic
parent
ca135802de
commit
3415301c07
|
|
@ -140,6 +140,8 @@ export default {
|
|||
|
||||
hand: {
|
||||
enabled: true,
|
||||
rotation: false, // use best-guess rotated hand image or just box with rotation as-is
|
||||
// false means higher performance, but incorrect finger mapping if hand is inverted
|
||||
inputSize: 256, // fixed value
|
||||
skipFrames: 19, // how many frames to go without re-running the hand bounding box detector
|
||||
// only used for video inputs
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@ import draw from './draw.js';
|
|||
import Menu from './menu.js';
|
||||
import GLBench from './gl-bench.js';
|
||||
|
||||
// const userConfig = {}; // add any user configuration overrides
|
||||
const userConfig = {}; // add any user configuration overrides
|
||||
/*
|
||||
const userConfig = {
|
||||
async: false,
|
||||
face: { enabled: false },
|
||||
body: { enabled: false },
|
||||
};
|
||||
*/
|
||||
|
||||
const human = new Human(userConfig);
|
||||
|
||||
|
|
@ -33,7 +37,7 @@ const ui = {
|
|||
console: true,
|
||||
maxFPSframes: 10,
|
||||
modelsPreload: true,
|
||||
modelsWarmup: false,
|
||||
modelsWarmup: true,
|
||||
menuWidth: 0,
|
||||
menuHeight: 0,
|
||||
camera: {},
|
||||
|
|
@ -44,7 +48,7 @@ const ui = {
|
|||
detectThread: null,
|
||||
framesDraw: 0,
|
||||
framesDetect: 0,
|
||||
bench: true,
|
||||
bench: false,
|
||||
};
|
||||
|
||||
// global variables
|
||||
|
|
@ -471,6 +475,10 @@ function setupMenu() {
|
|||
human.config.face.detector.iouThreshold = parseFloat(val);
|
||||
human.config.hand.iouThreshold = parseFloat(val);
|
||||
});
|
||||
menu.process.addBool('detection rotation', human.config.face.detector, 'rotation', (val) => {
|
||||
human.config.face.detector.rotation = val;
|
||||
human.config.hand.rotation = val;
|
||||
});
|
||||
menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
|
||||
menu.process.addButton('process sample images', 'process images', () => detectSampleImages());
|
||||
menu.process.addHTML('<hr style="border-style: inset; border-color: dimgray">');
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytes": 1786626,
|
||||
"bytes": 1786155,
|
||||
"imports": []
|
||||
},
|
||||
"demo/draw.js": {
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"demo/browser.js": {
|
||||
"bytes": 25223,
|
||||
"bytes": 25469,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/human.esm.js"
|
||||
|
|
@ -38,14 +38,14 @@
|
|||
"dist/demo-browser-index.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 2746890
|
||||
"bytes": 2747800
|
||||
},
|
||||
"dist/demo-browser-index.js": {
|
||||
"imports": [],
|
||||
"exports": [],
|
||||
"inputs": {
|
||||
"dist/human.esm.js": {
|
||||
"bytesInOutput": 1779386
|
||||
"bytesInOutput": 1778915
|
||||
},
|
||||
"demo/draw.js": {
|
||||
"bytesInOutput": 7816
|
||||
|
|
@ -57,10 +57,10 @@
|
|||
"bytesInOutput": 7382
|
||||
},
|
||||
"demo/browser.js": {
|
||||
"bytesInOutput": 19411
|
||||
"bytesInOutput": 19563
|
||||
}
|
||||
},
|
||||
"bytes": 1833184
|
||||
"bytes": 1832865
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -36,7 +36,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 13817,
|
||||
"bytes": 13856,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
|
|
@ -279,7 +279,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7607,
|
||||
"bytes": 7923,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
|
|
@ -289,6 +289,9 @@
|
|||
},
|
||||
{
|
||||
"path": "src/hand/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -297,7 +300,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytes": 3194,
|
||||
"bytes": 3250,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -339,7 +342,7 @@
|
|||
]
|
||||
},
|
||||
"config.js": {
|
||||
"bytes": 8779,
|
||||
"bytes": 8990,
|
||||
"imports": []
|
||||
},
|
||||
"src/sample.js": {
|
||||
|
|
@ -351,7 +354,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 16523,
|
||||
"bytes": 16524,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -405,7 +408,7 @@
|
|||
"dist/human.esm.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 2651614
|
||||
"bytes": 2652161
|
||||
},
|
||||
"dist/human.esm.js": {
|
||||
"imports": [],
|
||||
|
|
@ -426,7 +429,7 @@
|
|||
"bytesInOutput": 30817
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 9297
|
||||
"bytesInOutput": 9323
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"bytesInOutput": 2182
|
||||
|
|
@ -483,13 +486,13 @@
|
|||
"bytesInOutput": 2730
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 4624
|
||||
"bytesInOutput": 4456
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127032
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytesInOutput": 1953
|
||||
"bytesInOutput": 2007
|
||||
},
|
||||
"src/gesture/gesture.js": {
|
||||
"bytesInOutput": 2463
|
||||
|
|
@ -510,13 +513,13 @@
|
|||
"bytesInOutput": 10905
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"bytesInOutput": 1868
|
||||
"bytesInOutput": 1473
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"bytesInOutput": 1796
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1428
|
||||
"bytesInOutput": 1440
|
||||
},
|
||||
"src/sample.js": {
|
||||
"bytesInOutput": 11646
|
||||
|
|
@ -525,7 +528,7 @@
|
|||
"bytesInOutput": 21
|
||||
}
|
||||
},
|
||||
"bytes": 1786626
|
||||
"bytes": 1786155
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -36,7 +36,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytes": 13817,
|
||||
"bytes": 13856,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
|
|
@ -279,7 +279,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytes": 7607,
|
||||
"bytes": 7923,
|
||||
"imports": [
|
||||
{
|
||||
"path": "dist/tfjs.esm.js"
|
||||
|
|
@ -289,6 +289,9 @@
|
|||
},
|
||||
{
|
||||
"path": "src/hand/util.js"
|
||||
},
|
||||
{
|
||||
"path": "src/log.js"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -297,7 +300,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytes": 3194,
|
||||
"bytes": 3250,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -339,7 +342,7 @@
|
|||
]
|
||||
},
|
||||
"config.js": {
|
||||
"bytes": 8779,
|
||||
"bytes": 8990,
|
||||
"imports": []
|
||||
},
|
||||
"src/sample.js": {
|
||||
|
|
@ -351,7 +354,7 @@
|
|||
"imports": []
|
||||
},
|
||||
"src/human.js": {
|
||||
"bytes": 16523,
|
||||
"bytes": 16524,
|
||||
"imports": [
|
||||
{
|
||||
"path": "src/log.js"
|
||||
|
|
@ -405,7 +408,7 @@
|
|||
"dist/human.js.map": {
|
||||
"imports": [],
|
||||
"inputs": {},
|
||||
"bytes": 2668611
|
||||
"bytes": 2669160
|
||||
},
|
||||
"dist/human.js": {
|
||||
"imports": [],
|
||||
|
|
@ -424,7 +427,7 @@
|
|||
"bytesInOutput": 30817
|
||||
},
|
||||
"src/face/facepipeline.js": {
|
||||
"bytesInOutput": 9297
|
||||
"bytesInOutput": 9323
|
||||
},
|
||||
"src/face/facemesh.js": {
|
||||
"bytesInOutput": 2182
|
||||
|
|
@ -481,13 +484,13 @@
|
|||
"bytesInOutput": 2730
|
||||
},
|
||||
"src/hand/handpipeline.js": {
|
||||
"bytesInOutput": 4624
|
||||
"bytesInOutput": 4456
|
||||
},
|
||||
"src/hand/anchors.js": {
|
||||
"bytesInOutput": 127032
|
||||
},
|
||||
"src/hand/handpose.js": {
|
||||
"bytesInOutput": 1953
|
||||
"bytesInOutput": 2007
|
||||
},
|
||||
"src/gesture/gesture.js": {
|
||||
"bytesInOutput": 2463
|
||||
|
|
@ -508,13 +511,13 @@
|
|||
"bytesInOutput": 1520210
|
||||
},
|
||||
"src/hand/box.js": {
|
||||
"bytesInOutput": 1868
|
||||
"bytesInOutput": 1473
|
||||
},
|
||||
"src/hand/util.js": {
|
||||
"bytesInOutput": 1796
|
||||
},
|
||||
"config.js": {
|
||||
"bytesInOutput": 1428
|
||||
"bytesInOutput": 1440
|
||||
},
|
||||
"src/sample.js": {
|
||||
"bytesInOutput": 11646
|
||||
|
|
@ -523,7 +526,7 @@
|
|||
"bytesInOutput": 21
|
||||
}
|
||||
},
|
||||
"bytes": 1786700
|
||||
"bytes": 1786229
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -188,7 +188,8 @@ class Pipeline {
|
|||
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, rotatedImage, [this.meshHeight, this.meshWidth]).div(255);
|
||||
} else {
|
||||
rotationMatrix = util.IDENTITY_MATRIX;
|
||||
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, input, [this.meshHeight, this.meshWidth]).div(255);
|
||||
const cloned = input.clone();
|
||||
face = bounding.cutBoxFromImageAndResize({ startPoint: box.startPoint, endPoint: box.endPoint }, cloned, [this.meshHeight, this.meshWidth]).div(255);
|
||||
}
|
||||
|
||||
// if we're not going to produce mesh, don't spend time with further processing
|
||||
|
|
|
|||
|
|
@ -18,11 +18,13 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as box from './box';
|
||||
import * as util from './util';
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
import { log } from '../log.js';
|
||||
|
||||
const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
|
||||
const PALM_BOX_ENLARGE_FACTOR = 3;
|
||||
const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
|
||||
const HAND_BOX_ENLARGE_FACTOR = 1.65; // increased from model default 1.65;
|
||||
// const PALM_BOX_SHIFT_VECTOR = [0, -0.4];
|
||||
const PALM_BOX_ENLARGE_FACTOR = 5; // default 3
|
||||
// const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection
|
||||
const HAND_BOX_ENLARGE_FACTOR = 1.65; // default 1.65
|
||||
const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2];
|
||||
const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0;
|
||||
const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2;
|
||||
|
|
@ -38,22 +40,20 @@ class HandPipeline {
|
|||
}
|
||||
|
||||
getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
|
||||
const rotatedPalmLandmarks = palmLandmarks.map((coord) => {
|
||||
const homogeneousCoordinate = [...coord, 1];
|
||||
return util.rotatePoint(homogeneousCoordinate, rotationMatrix);
|
||||
});
|
||||
const rotatedPalmLandmarks = palmLandmarks.map((coord) => util.rotatePoint([...coord, 1], rotationMatrix));
|
||||
const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks);
|
||||
return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR);
|
||||
// return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR);
|
||||
return box.enlargeBox(box.squarifyBox(boxAroundPalm), PALM_BOX_ENLARGE_FACTOR);
|
||||
}
|
||||
|
||||
getBoxForHandLandmarks(landmarks) {
|
||||
const boundingBox = this.calculateLandmarksBoundingBox(landmarks);
|
||||
const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
|
||||
const palmLandmarks = [];
|
||||
// const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
|
||||
const boxAroundHand = box.enlargeBox(box.squarifyBox(boundingBox), HAND_BOX_ENLARGE_FACTOR);
|
||||
boxAroundHand.palmLandmarks = [];
|
||||
for (let i = 0; i < PALM_LANDMARK_IDS.length; i++) {
|
||||
palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
|
||||
boxAroundHand.palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2));
|
||||
}
|
||||
boxAroundHand.palmLandmarks = palmLandmarks;
|
||||
return boxAroundHand;
|
||||
}
|
||||
|
||||
|
|
@ -110,10 +110,10 @@ class HandPipeline {
|
|||
const currentBox = this.storedBoxes[i];
|
||||
if (!currentBox) continue;
|
||||
if (config.hand.landmarks) {
|
||||
const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]);
|
||||
const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]) : 0;
|
||||
const palmCenter = box.getBoxCenter(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
|
||||
const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized);
|
||||
const rotatedImage = config.hand.rotation ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone();
|
||||
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]);
|
||||
|
|
@ -146,7 +146,8 @@ class HandPipeline {
|
|||
}
|
||||
keypoints.dispose();
|
||||
} else {
|
||||
const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
|
||||
// const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR);
|
||||
const enlarged = box.enlargeBox(box.squarifyBox(currentBox), HAND_BOX_ENLARGE_FACTOR);
|
||||
const result = {
|
||||
confidence: currentBox.confidence,
|
||||
box: {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
import { log } from '../log.js';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as handdetector from './handdetector';
|
||||
import * as pipeline from './handpipeline';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import * as anchors from './anchors';
|
||||
|
||||
const MESH_ANNOTATIONS = {
|
||||
|
|
@ -32,8 +32,8 @@ const MESH_ANNOTATIONS = {
|
|||
};
|
||||
|
||||
class HandPose {
|
||||
constructor(pipe) {
|
||||
this.pipeline = pipe;
|
||||
constructor(handPipeline) {
|
||||
this.handPipeline = handPipeline;
|
||||
}
|
||||
|
||||
static getAnnotations() {
|
||||
|
|
@ -41,7 +41,7 @@ class HandPose {
|
|||
}
|
||||
|
||||
async estimateHands(input, config) {
|
||||
const predictions = await this.pipeline.estimateHands(input, config);
|
||||
const predictions = await this.handPipeline.estimateHands(input, config);
|
||||
if (!predictions) return [];
|
||||
const hands = [];
|
||||
for (const prediction of predictions) {
|
||||
|
|
@ -74,11 +74,11 @@ async function load(config) {
|
|||
config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null,
|
||||
config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null,
|
||||
]);
|
||||
const detector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors);
|
||||
const pipe = new pipeline.HandPipeline(detector, handPoseModel, config.hand.inputSize);
|
||||
const handpose = new HandPose(pipe);
|
||||
const handDetector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors);
|
||||
const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, config.hand.inputSize);
|
||||
const handPose = new HandPose(handPipeline);
|
||||
if (config.hand.enabled) log(`load model: ${config.hand.detector.modelPath.match(/\/(.*)\./)[1]}`);
|
||||
if (config.hand.landmarks) log(`load model: ${config.hand.skeleton.modelPath.match(/\/(.*)\./)[1]}`);
|
||||
return handpose;
|
||||
return handPose;
|
||||
}
|
||||
exports.load = load;
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ class Human {
|
|||
}
|
||||
this.firstRun = false;
|
||||
}
|
||||
|
||||
if (this.config.async) {
|
||||
[
|
||||
this.models.facemesh,
|
||||
|
|
|
|||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit 60eb01217f8d3e69055c991991183dd295cf3766
|
||||
Subproject commit 640fbd9a107c52692bfaaede0d751c5572cf7f22
|
||||
Loading…
Reference in New Issue