human/src/hand/handpose.js

/**
 * @license
 * Copyright 2020 Google LLC. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */
// https://storage.googleapis.com/tfjs-models/demos/handpose/index.html

const tf = require('@tensorflow/tfjs');
const handdetector = require('./handdetector');
const pipeline = require('./handpipeline');
const anchors = require('./anchors');

const MESH_ANNOTATIONS = {
  thumb: [1, 2, 3, 4],
  indexFinger: [5, 6, 7, 8],
  middleFinger: [9, 10, 11, 12],
  ringFinger: [13, 14, 15, 16],
  pinky: [17, 18, 19, 20],
  palmBase: [0],
};

class HandPose {
  constructor(pipe) {
    this.pipeline = pipe;
  }

  static getAnnotations() {
    return MESH_ANNOTATIONS;
  }

  async estimateHands(input, config) {
    const predictions = await this.pipeline.estimateHands(input, config);
    if (!predictions) return [];
    const hands = [];
    for (const prediction of predictions) {
      const annotations = {};
      if (prediction.landmarks) {
        for (const key of Object.keys(MESH_ANNOTATIONS)) {
          annotations[key] = MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
        }
      }
      hands.push({
        confidence: prediction.handInViewConfidence,
        box: prediction.boundingBox ? [
          prediction.boundingBox.topLeft[0],
          prediction.boundingBox.topLeft[1],
          prediction.boundingBox.bottomRight[0] - prediction.boundingBox.topLeft[0],
          prediction.boundingBox.bottomRight[1] - prediction.boundingBox.topLeft[1],
        ] : 0,
        landmarks: prediction.landmarks,
        annotations,
      });
    }
    return hands;
  }
}
exports.HandPose = HandPose;

async function load(config) {
  const [handDetectorModel, handPoseModel] = await Promise.all([
    tf.loadGraphModel(config.detector.modelPath, { fromTFHub: config.detector.modelPath.includes('tfhub.dev') }),
    tf.loadGraphModel(config.skeleton.modelPath, { fromTFHub: config.skeleton.modelPath.includes('tfhub.dev') }),
  ]);
  const detector = new handdetector.HandDetector(handDetectorModel, config.inputSize, anchors.anchors);
  const pipe = new pipeline.HandPipeline(detector, handPoseModel, config.inputSize);
  const handpose = new HandPose(pipe);
  return handpose;
}
exports.load = load;