From 0b907e56d96294b80a27b877ae7b6028b791378b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 22 May 2021 14:53:51 -0400 Subject: [PATCH] enhance strong typing --- CHANGELOG.md | 6 +++--- src/draw/draw.ts | 17 +++++++++-------- src/efficientpose/efficientpose.ts | 15 +++++++++------ src/face.ts | 2 +- src/handpose/handdetector.ts | 16 ++++++++-------- src/handpose/handpipeline.ts | 17 +++++++++-------- src/human.ts | 9 +++++---- src/object/nanodet.ts | 2 +- src/posenet/poses.ts | 2 +- src/posenet/utils.ts | 6 +++--- src/profile.ts | 2 +- src/result.ts | 7 +++++-- 12 files changed, 55 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc360421..1a015097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,14 @@ Repository: **** ## Changelog +### **HEAD -> main** 2021/05/22 mandic00@live.com + + ### **1.9.2** 2021/05/22 mandic00@live.com - add id and boxraw on missing objects - restructure results strong typing -### **origin/main** 2021/05/21 mandic00@live.com - - ### **1.9.1** 2021/05/21 mandic00@live.com - caching improvements diff --git a/src/draw/draw.ts b/src/draw/draw.ts index 73cfc56f..bbc8adb6 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -96,13 +96,14 @@ function rect(ctx, x, y, width, height, localOptions) { ctx.stroke(); } -function lines(ctx, points: [number, number, number][] = [], localOptions) { +function lines(ctx, points: [number, number, number?][] = [], localOptions) { if (points === undefined || points.length === 0) return; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); for (const pt of points) { - ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; - ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color; + const z = pt[2] || 0; + ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color; + ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color; ctx.lineTo(pt[0], Math.round(pt[1])); } ctx.stroke(); @@ -112,7 +113,7 @@ function lines(ctx, points: [number, number, number][] = [], localOptions) { } } -function curves(ctx, points: [number, number, number][] = [], localOptions) { +function curves(ctx, points: [number, number, number?][] = [], localOptions) { if (points === undefined || points.length === 0) return; if (!localOptions.useCurves || points.length <= 2) { lines(ctx, points, localOptions); @@ -142,8 +143,8 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array 1) && (what[1].length > 0)) { const person = where[1] > 0 ? `#${where[1]}` : ''; @@ -271,7 +272,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, dra } if (localOptions.drawPoints) { for (let pt = 0; pt < result[i].keypoints.length; pt++) { - ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : localOptions.color; + ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color; point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions); } } @@ -286,7 +287,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array, dra } if (localOptions.drawPolygons && result[i].keypoints) { let part; - const points: any[] = []; + const points: [number, number, number?][] = []; // shoulder line points.length = 0; part = result[i].keypoints.find((a) => a.part === 'leftShoulder'); diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index 0e89a7f9..abd2181c 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -3,7 +3,10 @@ import * as tf from '../../dist/tfjs.esm.js'; import { Body } from '../result'; let model; -let keypoints: Array = []; + +type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; + +let keypoints: Array = []; let skipped = Number.MAX_SAFE_INTEGER; const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle']; @@ -41,7 +44,8 @@ function max2d(inputs, minScore) { export async function predict(image, config): Promise { if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) { skipped++; - return keypoints; + const score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); + return [{ id: 0, score, keypoints }]; } skipped = 0; return new Promise(async (resolve) => { @@ -57,7 +61,7 @@ export async function predict(image, config): Promise { tensor.dispose(); if (resT) { - const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = []; + const parts: Array = []; const squeeze = resT.squeeze(); tf.dispose(resT); // body parts are basically just a stack of 2d tensors @@ -69,12 +73,11 @@ export async function predict(image, config): Promise { const [x, y, score] = max2d(stack[id], config.body.minConfidence); if (score > config.body.minConfidence) { parts.push({ - id, score: Math.round(100 * score) / 100, part: bodyParts[id], positionRaw: { - xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1 - yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1 + x: x / model.inputs[0].shape[2], // x normalized to 0..1 + y: y / model.inputs[0].shape[1], // y normalized to 0..1 }, position: { x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size diff --git a/src/face.ts b/src/face.ts index 2fee4ac2..dd4901c2 100644 --- a/src/face.ts +++ b/src/face.ts @@ -96,7 +96,7 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu return { angle, matrix }; }; -export const detectFace = async (parent, input): Promise => { +export const detectFace = async (parent, input): Promise => { // run facemesh, includes blazeface and iris // eslint-disable-next-line no-async-promise-executor let timeStamp; diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index e89e4bbc..82ba2e86 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -3,12 +3,12 @@ import * as box from './box'; import * as anchors from './anchors'; export class HandDetector { - model: any; - anchors: any; - anchorsTensor: any; + model: any; // tf.GraphModel + anchors: number[][]; + anchorsTensor: typeof tf.Tensor; inputSize: number; - inputSizeTensor: any; - doubleInputSizeTensor: any; + inputSizeTensor: typeof tf.Tensor; + doubleInputSizeTensor: typeof tf.Tensor; constructor(model) { this.model = model; @@ -52,7 +52,7 @@ export class HandDetector { scoresT.dispose(); filteredT.dispose(); - const hands: Array<{ box: any, palmLandmarks: any, confidence: number }> = []; + const hands: Array<{ box: any, palmLandmarks: any, confidence: number }> = []; // box and lardmarks are tensors here for (const index of filtered) { if (scores[index] >= config.hand.minConfidence) { const matchingBox = tf.slice(boxes, [index, 0], [1, -1]); @@ -67,13 +67,13 @@ export class HandDetector { return hands; } - async estimateHandBounds(input, config) { + async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> { const inputHeight = input.shape[1]; const inputWidth = input.shape[2]; const image = tf.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1)); const predictions = await this.getBoxes(image, config); image.dispose(); - const hands: Array<{}> = []; + const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = []; if (!predictions || predictions.length === 0) return hands; for (const prediction of predictions) { const boxes = prediction.box.dataSync(); diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index 8bb9e7e7..75b3af8c 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -1,6 +1,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as util from './util'; +import * as detector from './handdetector'; const palmBoxEnlargeFactor = 5; // default 3 const handBoxEnlargeFactor = 1.65; // default 1.65 @@ -9,17 +10,17 @@ const palmLandmarksPalmBase = 0; const palmLandmarksMiddleFingerBase = 2; export class HandPipeline { - handDetector: any; - landmarkDetector: any; + handDetector: detector.HandDetector; + handPoseModel: any; // tf.GraphModel inputSize: number; - storedBoxes: any; + storedBoxes: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number } | null>; skipped: number; detectedHands: number; - constructor(handDetector, landmarkDetector) { + constructor(handDetector, handPoseModel) { this.handDetector = handDetector; - this.landmarkDetector = landmarkDetector; - this.inputSize = this.landmarkDetector?.inputs[0].shape[2]; + this.handPoseModel = handPoseModel; + this.inputSize = this.handPoseModel?.inputs[0].shape[2]; this.storedBoxes = []; this.skipped = 0; this.detectedHands = 0; @@ -112,7 +113,7 @@ export class HandPipeline { const handImage = croppedInput.div(255); croppedInput.dispose(); rotatedImage.dispose(); - const [confidenceT, keypoints] = await this.landmarkDetector.predict(handImage); + const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage); handImage.dispose(); const confidence = confidenceT.dataSync()[0]; confidenceT.dispose(); @@ -123,7 +124,7 @@ export class HandPipeline { keypointsReshaped.dispose(); const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); const nextBoundingBox = this.getBoxForHandLandmarks(coords); - this.storedBoxes[i] = nextBoundingBox; + this.storedBoxes[i] = { ...nextBoundingBox, confidence }; const result = { landmarks: coords, confidence, diff --git a/src/human.ts b/src/human.ts index 0f2926b3..18921c6e 100644 --- a/src/human.ts +++ b/src/human.ts @@ -1,6 +1,6 @@ import { log, now, mergeDeep } from './helpers'; import { Config, defaults } from './config'; -import { Result } from './result'; +import { Result, Gesture } from './result'; import * as sysinfo from './sysinfo'; import * as tf from '../dist/tfjs.esm.js'; import * as backend from './tfjs/backend'; @@ -114,7 +114,7 @@ export class Human { /** Platform and agent information detected by Human */ sysinfo: { platform: string, agent: string }; /** Performance object that contains values for all recently performed operations */ - perf: any; + perf: any; // perf members are dynamically defined as needed #numTensors: number; #analyzeMemoryLeaks: boolean; #checkSanity: boolean; @@ -449,9 +449,10 @@ export class Human { this.analyze('Check Changed:'); // prepare where to store model results + // keep them with weak typing as it can be promise or not + let faceRes; let bodyRes; let handRes; - let faceRes; let objectRes; let current; @@ -520,7 +521,7 @@ export class Human { tf.dispose(process.tensor); // run gesture analysis last - let gestureRes: any[] = []; + let gestureRes: Gesture[] = []; if (this.config.gesture.enabled) { timeStamp = now(); gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)]; diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 6531ae14..b4c5c372 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -81,7 +81,7 @@ async function process(res, inputSize, outputShape, config) { // unnecessary boxes and run nms only on good candidates (basically it just does IOU analysis as scores are already filtered) const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]); // switches coordinates from x,y to y,x as expected by tf.nms const nmsScores = results.map((a) => a.score); - let nmsIdx: any[] = []; + let nmsIdx: Array = []; if (nmsBoxes && nmsBoxes.length > 0) { const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); nmsIdx = nms.dataSync(); diff --git a/src/posenet/poses.ts b/src/posenet/poses.ts index e1a0dcb1..296f2926 100644 --- a/src/posenet/poses.ts +++ b/src/posenet/poses.ts @@ -120,7 +120,7 @@ function getInstanceScore(existingPoses, keypoints) { } export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) { - const poses: Array<{ keypoints: any, box: any, score: number }> = []; + const poses: Array<{ keypoints, box: [number, number, number, number], score: number }> = []; const queue = buildPartWithScoreQueue(minConfidence, scores); // Generate at most maxDetected object instances per image in decreasing root part score order. while (poses.length < maxDetected && !queue.empty()) { diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index a38dd8e0..7c7bb221 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -14,7 +14,7 @@ export function getAdjacentKeyPoints(keypoints, minConfidence) { }, []); } -export function getBoundingBox(keypoints) { +export function getBoundingBox(keypoints): [number, number, number, number] { const coord = keypoints.reduce(({ maxX, maxY, minX, minY }, { position: { x, y } }) => ({ maxX: Math.max(maxX, x), maxY: Math.max(maxY, y), @@ -49,9 +49,9 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input // algorithm based on Coursera Lecture from Algorithms, Part 1: https://www.coursera.org/learn/algorithms-part1/lecture/ZjoSM/heapsort export class MaxHeap { - priorityQueue: any; + priorityQueue: Array; // don't touch numberOfElements: number; - getElementValue: any; + getElementValue: any; // function call constructor(maxSize, getElementValue) { this.priorityQueue = new Array(maxSize); diff --git a/src/profile.ts b/src/profile.ts index 7390058a..f7e034c2 100644 --- a/src/profile.ts +++ b/src/profile.ts @@ -2,7 +2,7 @@ import { log } from './helpers'; export const data = {}; -export function run(modelName: string, profileData: any): void { +export function run(modelName: string, profileData: any): void { // profileData is tfjs internal type if (!profileData || !profileData.kernels) return; const maxDetected = 5; const time = profileData.kernels diff --git a/src/result.ts b/src/result.ts index d2b0064f..da1d0f81 100644 --- a/src/result.ts +++ b/src/result.ts @@ -30,6 +30,8 @@ * - matrix: 3d transofrmation matrix as array of numeric values * - tensor: face tensor as Tensor object which contains detected face */ +import { Tensor } from '../dist/tfjs.esm.js'; + export interface Face { id: number confidence: number, @@ -50,7 +52,7 @@ export interface Face { angle: { roll: number, yaw: number, pitch: number }, matrix: [number, number, number, number, number, number, number, number, number], } - tensor: any, + tensor: typeof Tensor, } /** Body results @@ -75,7 +77,8 @@ export interface Body { boxRaw?: [x: number, y: number, width: number, height: number], keypoints: Array<{ part: string, - position: { x: number, y: number, z: number }, + position: { x: number, y: number, z?: number }, + positionRaw?: { x: number, y: number, z?: number }, score: number, presence?: number, }>