ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

major work on handpose model

pull/293/head
Vladimir Mandic 2020-11-04 14:59:30 -05:00
parent ba0437cc8b
commit 967877bd76
6 changed files with 65 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
demo/browser.js
View File

@ -332,7 +332,7 @@ function setupMenu() {
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Model Parameters');
menu.addRange('Max Objects', config.face.detector, 'maxFaces', 0, 50, 1, (val) => {
menu.addRange('Max Objects', config.face.detector, 'maxFaces', 1, 50, 1, (val) => {
config.face.detector.maxFaces = parseInt(val);
config.body.maxDetections = parseInt(val);
config.hand.maxHands = parseInt(val);

8
src/gesture.js
View File

@ -37,9 +37,11 @@ exports.hand = (res) => {
for (const [finger, pos] of Object.entries(hand['annotations'])) {
if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger
}
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push(`${closest.name} forward ${highest.name} up`);
if (fingers && fingers.length > 0) {
const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a));
const highest = fingers.reduce((best, a) => (best.position[1] < a.position[1] ? best : a));
gestures.push(`${closest.name} forward ${highest.name} up`);
}
}
return gestures;
};

25
src/handpose/handdetector.js
View File

@ -21,8 +21,6 @@ const box = require('./box');
class HandDetector {
constructor(model, inputSize, anchorsAnnotated) {
this.model = model;
this.width = inputSize;
this.height = inputSize;
this.anchors = anchorsAnnotated.map((anchor) => [anchor.x_center, anchor.y_center]);
this.anchorsTensor = tf.tensor2d(this.anchors);
this.inputSizeTensor = tf.tensor1d([inputSize, inputSize]);
@ -49,16 +47,14 @@ class HandDetector {
}
async getBoundingBoxes(input, config) {
const normalizedInput = tf.tidy(() => tf.mul(tf.sub(input, 0.5), 2));
const batchedPrediction = this.model.predict(normalizedInput);
const batchedPrediction = this.model.predict(input);
const prediction = batchedPrediction.squeeze();
const scores = tf.tidy(() => tf.sigmoid(tf.slice(prediction, [0, 0], [-1, 1])).squeeze());
const rawBoxes = tf.slice(prediction, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
const boxesWithHandsTensor = tf.image.nonMaxSuppression(boxes, scores, config.maxHands, config.iouThreshold, config.scoreThreshold);
const boxesWithHandsTensor = tf.image.nonMaxSuppression(boxes, scores, config.maxHands, config.iouThreshold, 0.95); // config.scoreThreshold
const boxesWithHands = boxesWithHandsTensor.arraySync();
const toDispose = [
normalizedInput,
batchedPrediction,
boxesWithHandsTensor,
prediction,
@ -85,22 +81,19 @@ class HandDetector {
async estimateHandBounds(input, config) {
const inputHeight = input.shape[1];
const inputWidth = input.shape[2];
const image = tf.tidy(() => input.resizeBilinear([this.width, this.height]).div(255));
const image = tf.tidy(() => input.resizeBilinear([config.inputSize, config.inputSize]).div(127.5).sub(1));
const predictions = await this.getBoundingBoxes(image, config);
if (!predictions || predictions.length === 0) {
image.dispose();
return null;
}
image.dispose();
if (!predictions || predictions.length === 0) return null;
const hands = [];
for (const prediction of predictions) {
const boundingBoxes = prediction.boxes.arraySync();
const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4);
const boundingBoxes = prediction.boxes.dataSync();
const startPoint = boundingBoxes.slice(0, 2);
const endPoint = boundingBoxes.slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync();
image.dispose();
prediction.boxes.dispose();
prediction.palmLandmarks.dispose();
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / this.width, inputHeight / this.height]));
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks }, [inputWidth / config.inputSize, inputHeight / config.inputSize]));
}
return hands;
}

62
src/handpose/handpipeline.js
View File

@ -35,8 +35,8 @@ class HandPipeline {
this.inputSize = inputSize;
this.regionsOfInterest = [];
this.runsWithoutHandDetector = 0;
this.maxHandsNumber = 1;
this.skipFrames = 0;
this.detectedHands = 0;
}
getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) {
@ -87,12 +87,18 @@ class HandPipeline {
async estimateHands(image, config) {
this.skipFrames = config.skipFrames;
const useFreshBox = this.shouldUpdateRegionsOfInterest();
// don't need box detection if we have sufficient number of boxes
let useFreshBox = (this.detectedHands === 0) || (this.detectedHands !== this.regionsOfInterest.length);
let boundingBoxPredictions;
// but every skipFrames check if detect boxes number changed
if (useFreshBox || this.runsWithoutHandDetector > this.skipFrames) boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
// if there are new boxes and number of boxes doesn't match use new boxes, but not if maxhands is fixed to 1
if (config.maxHands > 1 && boundingBoxPredictions && boundingBoxPredictions.length > 0 && boundingBoxPredictions.length !== this.detectedHands) useFreshBox = true;
if (useFreshBox) {
const boundingBoxPredictions = await this.boundingBoxDetector.estimateHandBounds(image, config);
this.regionsOfInterest = [];
if (!boundingBoxPredictions || boundingBoxPredictions.length === 0) {
image.dispose();
this.detectedHands = 0;
return null;
}
for (const boundingBoxPrediction of boundingBoxPredictions) {
@ -121,28 +127,38 @@ class HandPipeline {
handImage.dispose();
const confidenceValue = confidence.dataSync()[0];
confidence.dispose();
if (confidenceValue < config.minConfidence) {
if (confidenceValue >= config.minConfidence) {
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
this.regionsOfInterest[i] = null;
return null;
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateRegionsOfInterest(nextBoundingBox, i);
const result = {
landmarks: coords,
handInViewConfidence: confidenceValue,
boundingBox: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint,
},
};
hands.push(result);
} else {
/*
const result = {
handInViewConfidence: confidenceValue,
boundingBox: {
topLeft: currentBox.startPoint,
bottomRight: currentBox.endPoint,
},
};
hands.push(result);
*/
}
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
const rawCoords = keypointsReshaped.arraySync();
keypoints.dispose();
keypointsReshaped.dispose();
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
this.updateRegionsOfInterest(nextBoundingBox, i);
const result = {
landmarks: coords,
handInViewConfidence: confidenceValue,
boundingBox: {
topLeft: nextBoundingBox.startPoint,
bottomRight: nextBoundingBox.endPoint,
},
};
hands.push(result);
}
this.detectedHands = hands.length;
return hands;
}
@ -174,10 +190,6 @@ class HandPipeline {
}
this.regionsOfInterest[i] = iou > UPDATE_REGION_OF_INTEREST_IOU_THRESHOLD ? previousBox : newBox;
}
shouldUpdateRegionsOfInterest() {
return !this.regionsOfInterest || (this.regionsOfInterest.length === 0) || (this.runsWithoutHandDetector >= this.skipFrames);
}
}
exports.HandPipeline = HandPipeline;

15
src/handpose/handpose.js
View File

@ -14,6 +14,8 @@
* limitations under the License.
* =============================================================================
*/
// https://storage.googleapis.com/tfjs-models/demos/handpose/index.html
const tf = require('@tensorflow/tfjs');
const handdetector = require('./handdetector');
const pipeline = require('./handpipeline');
@ -43,12 +45,19 @@ class HandPose {
const hands = [];
for (const prediction of predictions) {
const annotations = {};
for (const key of Object.keys(MESH_ANNOTATIONS)) {
annotations[key] = MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
if (prediction.landmarks) {
for (const key of Object.keys(MESH_ANNOTATIONS)) {
annotations[key] = MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
}
}
hands.push({
confidence: prediction.handInViewConfidence,
box: prediction.boundingBox ? [prediction.boundingBox.topLeft[0], prediction.boundingBox.topLeft[1], prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0], prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1]] : 0,
box: prediction.boundingBox ? [
prediction.boundingBox.topLeft[0],
prediction.boundingBox.topLeft[1],
prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0],
prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1],
] : 0,
landmarks: prediction.landmarks,
annotations,
});

2
src/human.js
View File

@ -171,7 +171,7 @@ class Human {
}
// tf.ENV.set('WEBGL_CPU_FORWARD', true);
// tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
// tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
await tf.ready();
}
}