From 565a8b116a14480a18c2a279aa2960103f9b6049 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 10 Dec 2020 15:46:45 -0500 Subject: [PATCH] conditional hand rotation --- config.js | 2 ++ demo/browser.js | 14 +++++++++++--- src/hand/handpipeline.js | 33 +++++++++++++++++---------------- src/hand/handpose.js | 16 ++++++++-------- src/human.js | 1 + wiki | 2 +- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/config.js b/config.js index c94e7a41..15fe3ee6 100644 --- a/config.js +++ b/config.js @@ -140,6 +140,8 @@ export default { hand: { enabled: true, + rotation: false, // use best-guess rotated hand image or just box with rotation as-is + // false means higher performance, but incorrect finger mapping if hand is inverted inputSize: 256, // fixed value skipFrames: 19, // how many frames to go without re-running the hand bounding box detector // only used for video inputs diff --git a/demo/browser.js b/demo/browser.js index cca63479..c0e2556b 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -3,10 +3,14 @@ import draw from './draw.js'; import Menu from './menu.js'; import GLBench from './gl-bench.js'; -// const userConfig = {}; // add any user configuration overrides +const userConfig = {}; // add any user configuration overrides +/* const userConfig = { async: false, + face: { enabled: false }, + body: { enabled: false }, }; +*/ const human = new Human(userConfig); @@ -33,7 +37,7 @@ const ui = { console: true, maxFPSframes: 10, modelsPreload: true, - modelsWarmup: false, + modelsWarmup: true, menuWidth: 0, menuHeight: 0, camera: {}, @@ -44,7 +48,7 @@ const ui = { detectThread: null, framesDraw: 0, framesDetect: 0, - bench: true, + bench: false, }; // global variables @@ -471,6 +475,10 @@ function setupMenu() { human.config.face.detector.iouThreshold = parseFloat(val); human.config.hand.iouThreshold = parseFloat(val); }); + menu.process.addBool('detection rotation', human.config.face.detector, 'rotation', (val) => { + human.config.face.detector.rotation = val; + human.config.hand.rotation = val; + }); menu.process.addHTML('
'); menu.process.addButton('process sample images', 'process images', () => detectSampleImages()); menu.process.addHTML('
'); diff --git a/src/hand/handpipeline.js b/src/hand/handpipeline.js index 49bb7b0e..79a3e1f9 100644 --- a/src/hand/handpipeline.js +++ b/src/hand/handpipeline.js @@ -18,11 +18,13 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as util from './util'; +// eslint-disable-next-line no-unused-vars +import { log } from '../log.js'; -const PALM_BOX_SHIFT_VECTOR = [0, -0.4]; -const PALM_BOX_ENLARGE_FACTOR = 3; -const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection -const HAND_BOX_ENLARGE_FACTOR = 1.65; // increased from model default 1.65; +// const PALM_BOX_SHIFT_VECTOR = [0, -0.4]; +const PALM_BOX_ENLARGE_FACTOR = 5; // default 3 +// const HAND_BOX_SHIFT_VECTOR = [0, -0.1]; // move detected hand box by x,y to ease landmark detection +const HAND_BOX_ENLARGE_FACTOR = 1.65; // default 1.65 const PALM_LANDMARK_IDS = [0, 5, 9, 13, 17, 1, 2]; const PALM_LANDMARKS_INDEX_OF_PALM_BASE = 0; const PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE = 2; @@ -38,22 +40,20 @@ class HandPipeline { } getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) { - const rotatedPalmLandmarks = palmLandmarks.map((coord) => { - const homogeneousCoordinate = [...coord, 1]; - return util.rotatePoint(homogeneousCoordinate, rotationMatrix); - }); + const rotatedPalmLandmarks = palmLandmarks.map((coord) => util.rotatePoint([...coord, 1], rotationMatrix)); const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks); - return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR); + // return box.enlargeBox(box.squarifyBox(box.shiftBox(boxAroundPalm, PALM_BOX_SHIFT_VECTOR)), PALM_BOX_ENLARGE_FACTOR); + return box.enlargeBox(box.squarifyBox(boxAroundPalm), PALM_BOX_ENLARGE_FACTOR); } getBoxForHandLandmarks(landmarks) { const boundingBox = this.calculateLandmarksBoundingBox(landmarks); - const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR); - const palmLandmarks = []; + // const boxAroundHand = box.enlargeBox(box.squarifyBox(box.shiftBox(boundingBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR); + const boxAroundHand = box.enlargeBox(box.squarifyBox(boundingBox), HAND_BOX_ENLARGE_FACTOR); + boxAroundHand.palmLandmarks = []; for (let i = 0; i < PALM_LANDMARK_IDS.length; i++) { - palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2)); + boxAroundHand.palmLandmarks.push(landmarks[PALM_LANDMARK_IDS[i]].slice(0, 2)); } - boxAroundHand.palmLandmarks = palmLandmarks; return boxAroundHand; } @@ -110,10 +110,10 @@ class HandPipeline { const currentBox = this.storedBoxes[i]; if (!currentBox) continue; if (config.hand.landmarks) { - const angle = util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]); + const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_PALM_BASE], currentBox.palmLandmarks[PALM_LANDMARKS_INDEX_OF_MIDDLE_FINGER_BASE]) : 0; const palmCenter = box.getBoxCenter(currentBox); const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]]; - const rotatedImage = tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized); + const rotatedImage = config.hand.rotation ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone(); const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter); const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]); @@ -146,7 +146,8 @@ class HandPipeline { } keypoints.dispose(); } else { - const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR); + // const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), HAND_BOX_ENLARGE_FACTOR); + const enlarged = box.enlargeBox(box.squarifyBox(currentBox), HAND_BOX_ENLARGE_FACTOR); const result = { confidence: currentBox.confidence, box: { diff --git a/src/hand/handpose.js b/src/hand/handpose.js index cf8d99c5..a4b34c7a 100644 --- a/src/hand/handpose.js +++ b/src/hand/handpose.js @@ -19,7 +19,7 @@ import { log } from '../log.js'; import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; -import * as pipeline from './handpipeline'; +import * as handpipeline from './handpipeline'; import * as anchors from './anchors'; const MESH_ANNOTATIONS = { @@ -32,8 +32,8 @@ const MESH_ANNOTATIONS = { }; class HandPose { - constructor(pipe) { - this.pipeline = pipe; + constructor(handPipeline) { + this.handPipeline = handPipeline; } static getAnnotations() { @@ -41,7 +41,7 @@ class HandPose { } async estimateHands(input, config) { - const predictions = await this.pipeline.estimateHands(input, config); + const predictions = await this.handPipeline.estimateHands(input, config); if (!predictions) return []; const hands = []; for (const prediction of predictions) { @@ -74,11 +74,11 @@ async function load(config) { config.hand.enabled ? tf.loadGraphModel(config.hand.detector.modelPath, { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null, config.hand.landmarks ? tf.loadGraphModel(config.hand.skeleton.modelPath, { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null, ]); - const detector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors); - const pipe = new pipeline.HandPipeline(detector, handPoseModel, config.hand.inputSize); - const handpose = new HandPose(pipe); + const handDetector = new handdetector.HandDetector(handDetectorModel, config.hand.inputSize, anchors.anchors); + const handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel, config.hand.inputSize); + const handPose = new HandPose(handPipeline); if (config.hand.enabled) log(`load model: ${config.hand.detector.modelPath.match(/\/(.*)\./)[1]}`); if (config.hand.landmarks) log(`load model: ${config.hand.skeleton.modelPath.match(/\/(.*)\./)[1]}`); - return handpose; + return handPose; } exports.load = load; diff --git a/src/human.js b/src/human.js index f816b17b..92e6f54c 100644 --- a/src/human.js +++ b/src/human.js @@ -120,6 +120,7 @@ class Human { } this.firstRun = false; } + if (this.config.async) { [ this.models.facemesh, diff --git a/wiki b/wiki index 60eb0121..640fbd9a 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 60eb01217f8d3e69055c991991183dd295cf3766 +Subproject commit 640fbd9a107c52692bfaaede0d751c5572cf7f22