diff --git a/CHANGELOG.md b/CHANGELOG.md index fd54fc6b..80fd473b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,12 @@ ## Changelog +### **HEAD -> main** 2021/10/19 mandic00@live.com + + ### **2.3.5** 2021/10/19 mandic00@live.com - -### **origin/main** 2021/10/19 snaggadhagga@gmail.com - +- removed direct usage of performance.now ### **2.3.4** 2021/10/19 mandic00@live.com diff --git a/package.json b/package.json index 050963e2..4f785440 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "@types/node": "^16.11.1", "@typescript-eslint/eslint-plugin": "^5.1.0", "@typescript-eslint/parser": "^5.1.0", - "@vladmandic/build": "^0.6.1", + "@vladmandic/build": "^0.6.2", "@vladmandic/pilogger": "^0.3.3", "canvas": "^2.8.0", "dayjs": "^1.10.7", diff --git a/src/handpose/handpose.ts b/src/hand/handpose.ts similarity index 97% rename from src/handpose/handpose.ts rename to src/hand/handpose.ts index 7c9b0a5f..f280415e 100644 --- a/src/handpose/handpose.ts +++ b/src/hand/handpose.ts @@ -6,9 +6,9 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; -import * as handdetector from './handdetector'; -import * as handpipeline from './handpipeline'; -import * as fingerPose from '../hand/fingerpose'; +import * as handdetector from './handposedetector'; +import * as handpipeline from './handposepipeline'; +import * as fingerPose from './fingerpose'; import type { HandResult, Box, Point } from '../result'; import type { Tensor, GraphModel } from '../tfjs/types'; import type { Config } from '../config'; diff --git a/src/handpose/anchors.ts b/src/hand/handposeanchors.ts similarity index 100% rename from src/handpose/anchors.ts rename to src/hand/handposeanchors.ts diff --git a/src/handpose/handdetector.ts b/src/hand/handposedetector.ts similarity index 93% rename from src/handpose/handdetector.ts rename to src/hand/handposedetector.ts index f78c9d26..656cbd68 100644 --- a/src/handpose/handdetector.ts +++ b/src/hand/handposedetector.ts @@ -4,8 +4,8 @@ */ import * as tf from '../../dist/tfjs.esm.js'; -import * as box from './box'; -import * as anchors from './anchors'; +import * as util from './handposeutil'; +import * as anchors from './handposeanchors'; import type { Tensor, GraphModel } from '../tfjs/types'; export class HandDetector { @@ -81,7 +81,7 @@ export class HandDetector { const palmLandmarks = await prediction.palmLandmarks.array(); tf.dispose(prediction.box); tf.dispose(prediction.palmLandmarks); - hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); + hands.push(util.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); } return hands; } diff --git a/src/handpose/handpipeline.ts b/src/hand/handposepipeline.ts similarity index 91% rename from src/handpose/handpipeline.ts rename to src/hand/handposepipeline.ts index 6872da73..fdb69d82 100644 --- a/src/handpose/handpipeline.ts +++ b/src/hand/handposepipeline.ts @@ -4,9 +4,8 @@ */ import * as tf from '../../dist/tfjs.esm.js'; -import * as box from './box'; -import * as util from './util'; -import type * as detector from './handdetector'; +import * as util from './handposeutil'; +import type * as detector from './handposedetector'; import type { Tensor, GraphModel } from '../tfjs/types'; import { env } from '../util/env'; @@ -45,12 +44,12 @@ export class HandPipeline { getBoxForPalmLandmarks(palmLandmarks, rotationMatrix) { const rotatedPalmLandmarks = palmLandmarks.map((coord) => util.rotatePoint([...coord, 1], rotationMatrix)); const boxAroundPalm = this.calculateLandmarksBoundingBox(rotatedPalmLandmarks); - return box.enlargeBox(box.squarifyBox(boxAroundPalm), palmBoxEnlargeFactor); + return util.enlargeBox(util.squarifyBox(boxAroundPalm), palmBoxEnlargeFactor); } getBoxForHandLandmarks(landmarks) { const boundingBox = this.calculateLandmarksBoundingBox(landmarks); - const boxAroundHand = box.enlargeBox(box.squarifyBox(boundingBox), handBoxEnlargeFactor); + const boxAroundHand = util.enlargeBox(util.squarifyBox(boundingBox), handBoxEnlargeFactor); boxAroundHand.palmLandmarks = []; for (let i = 0; i < palmLandmarkIds.length; i++) { boxAroundHand.palmLandmarks.push(landmarks[palmLandmarkIds[i]].slice(0, 2)); @@ -59,7 +58,7 @@ export class HandPipeline { } transformRawCoords(rawCoords, box2, angle, rotationMatrix) { - const boxSize = box.getBoxSize(box2); + const boxSize = util.getBoxSize(box2); const scaleFactor = [boxSize[0] / this.inputSize, boxSize[1] / this.inputSize, (boxSize[0] + boxSize[1]) / this.inputSize / 2]; const coordsScaled = rawCoords.map((coord) => [ scaleFactor[0] * (coord[0] - this.inputSize / 2), @@ -72,7 +71,7 @@ export class HandPipeline { return [...rotated, coord[2]]; }); const inverseRotationMatrix = util.invertTransformMatrix(rotationMatrix); - const boxCenter = [...box.getBoxCenter(box2), 1]; + const boxCenter = [...util.getBoxCenter(box2), 1]; const originalBoxCenter = [ util.dot(boxCenter, inverseRotationMatrix[0]), util.dot(boxCenter, inverseRotationMatrix[1]), @@ -112,12 +111,12 @@ export class HandPipeline { if (!currentBox) continue; if (config.hand.landmarks) { const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0; - const palmCenter = box.getBoxCenter(currentBox); + const palmCenter = util.getBoxCenter(currentBox); const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]]; const rotatedImage = config.hand.rotation && env.kernels.includes('rotatewithoffset') ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone(); const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter); const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox; - const croppedInput = box.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]); + const croppedInput = util.cutBoxFromImageAndResize(newBox, rotatedImage, [this.inputSize, this.inputSize]); const handImage = tf.div(croppedInput, 255); tf.dispose(croppedInput); tf.dispose(rotatedImage); @@ -148,7 +147,7 @@ export class HandPipeline { tf.dispose(keypoints); } else { // const enlarged = box.enlargeBox(box.squarifyBox(box.shiftBox(currentBox, HAND_BOX_SHIFT_VECTOR)), handBoxEnlargeFactor); - const enlarged = box.enlargeBox(box.squarifyBox(currentBox), handBoxEnlargeFactor); + const enlarged = util.enlargeBox(util.squarifyBox(currentBox), handBoxEnlargeFactor); const result = { confidence: currentBox.confidence, boxConfidence: currentBox.confidence, diff --git a/src/handpose/box.ts b/src/hand/handposeutil.ts similarity index 51% rename from src/handpose/box.ts rename to src/hand/handposeutil.ts index 79e4ca18..847b2c99 100644 --- a/src/handpose/box.ts +++ b/src/hand/handposeutil.ts @@ -1,8 +1,3 @@ -/** - * HandPose model implementation - * See `handpose.ts` for entry point - */ - import * as tf from '../../dist/tfjs.esm.js'; export function getBoxSize(box) { @@ -70,3 +65,73 @@ export function shiftBox(box, shiftFactor) { const endPoint = [box.endPoint[0] + shiftVector[0], box.endPoint[1] + shiftVector[1]]; return { startPoint, endPoint, palmLandmarks: box.palmLandmarks }; } + +export function normalizeRadians(angle) { + return angle - 2 * Math.PI * Math.floor((angle + Math.PI) / (2 * Math.PI)); +} + +export function computeRotation(point1, point2) { + const radians = Math.PI / 2 - Math.atan2(-(point2[1] - point1[1]), point2[0] - point1[0]); + return normalizeRadians(radians); +} + +export const buildTranslationMatrix = (x, y) => [[1, 0, x], [0, 1, y], [0, 0, 1]]; + +export function dot(v1, v2) { + let product = 0; + for (let i = 0; i < v1.length; i++) { + product += v1[i] * v2[i]; + } + return product; +} + +export function getColumnFrom2DArr(arr, columnIndex) { + const column: Array = []; + for (let i = 0; i < arr.length; i++) { + column.push(arr[i][columnIndex]); + } + return column; +} + +export function multiplyTransformMatrices(mat1, mat2) { + const product: Array = []; + const size = mat1.length; + for (let row = 0; row < size; row++) { + product.push([]); + for (let col = 0; col < size; col++) { + product[row].push(dot(mat1[row], getColumnFrom2DArr(mat2, col))); + } + } + return product; +} + +export function buildRotationMatrix(rotation, center) { + const cosA = Math.cos(rotation); + const sinA = Math.sin(rotation); + const rotationMatrix = [[cosA, -sinA, 0], [sinA, cosA, 0], [0, 0, 1]]; + const translationMatrix = buildTranslationMatrix(center[0], center[1]); + const translationTimesRotation = multiplyTransformMatrices(translationMatrix, rotationMatrix); + const negativeTranslationMatrix = buildTranslationMatrix(-center[0], -center[1]); + return multiplyTransformMatrices(translationTimesRotation, negativeTranslationMatrix); +} + +export function invertTransformMatrix(matrix) { + const rotationComponent = [[matrix[0][0], matrix[1][0]], [matrix[0][1], matrix[1][1]]]; + const translationComponent = [matrix[0][2], matrix[1][2]]; + const invertedTranslation = [ + -dot(rotationComponent[0], translationComponent), + -dot(rotationComponent[1], translationComponent), + ]; + return [ + rotationComponent[0].concat(invertedTranslation[0]), + rotationComponent[1].concat(invertedTranslation[1]), + [0, 0, 1], + ]; +} + +export function rotatePoint(homogeneousCoordinate, rotationMatrix) { + return [ + dot(homogeneousCoordinate, rotationMatrix[0]), + dot(homogeneousCoordinate, rotationMatrix[1]), + ]; +} diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index 76f37ae7..488e4f05 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -219,6 +219,11 @@ export async function predict(input: Tensor, config: Config): Promise