human/src/handpose/handpose.js

const tf = require('@tensorflow/tfjs');
const hand = require('./handdetector');
const keypoints = require('./keypoints');
const pipe = require('./pipeline');

class HandPose {
  constructor(pipeline) {
    this.pipeline = pipeline;
  }

  async estimateHands(input, config) {
    this.skipFrames = config.skipFrames;
    this.detectionConfidence = config.minConfidence;
    this.maxHands = config.maxHands;
    const image = tf.tidy(() => {
      if (!(input instanceof tf.Tensor)) {
        input = tf.browser.fromPixels(input);
      }
      return input.toFloat().expandDims(0);
    });
    const predictions = await this.pipeline.estimateHands(image, config);
    image.dispose();
    const hands = [];
    if (!predictions) return hands;
    for (const prediction of predictions) {
      if (!prediction) return [];
      const annotations = {};
      for (const key of Object.keys(keypoints.MESH_ANNOTATIONS)) {
        annotations[key] = keypoints.MESH_ANNOTATIONS[key].map((index) => prediction.landmarks[index]);
      }
      hands.push({
        confidence: prediction.confidence || 0,
        box: prediction.box ? [prediction.box.topLeft[0], prediction.box.topLeft[1], prediction.box.bottomRight[0] - prediction.box.topLeft[0], prediction.box.bottomRight[1] - prediction.box.topLeft[1]] : 0,
        landmarks: prediction.landmarks,
        annotations,
      });
    }
    return hands;
  }
}
exports.HandPose = HandPose;

async function loadAnchors(url) {
  if (tf.env().features.IS_NODE) {
    // eslint-disable-next-line global-require
    const fs = require('fs');
    const data = await fs.readFileSync(url.replace('file://', ''));
    return JSON.parse(data);
  }
  return tf.util.fetch(url).then((d) => d.json());
}

async function load(config) {
  const [anchors, handDetectorModel, handPoseModel] = await Promise.all([
    loadAnchors(config.detector.anchors),
    tf.loadGraphModel(config.detector.modelPath, { fromTFHub: config.detector.modelPath.includes('tfhub.dev') }),
    tf.loadGraphModel(config.skeleton.modelPath, { fromTFHub: config.skeleton.modelPath.includes('tfhub.dev') }),
  ]);
  const detector = new hand.HandDetector(handDetectorModel, anchors, config);
  const pipeline = new pipe.HandPipeline(detector, handPoseModel, config);
  const handpose = new HandPose(pipeline);
  return handpose;
}
exports.load = load;