diff --git a/CHANGELOG.md b/CHANGELOG.md index 11d1064a..3779468d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,11 @@ ## Changelog -### **HEAD -> main** 2021/09/25 mandic00@live.com - - -### **origin/main** 2021/09/25 mandic00@live.com +### **HEAD -> main** 2021/09/27 mandic00@live.com +- autodetect number of bodies and hands +- upload new samples +- new samples gallery and major code folder restructure - new release ### **2.2.3** 2021/09/24 mandic00@live.com diff --git a/src/draw.ts b/src/draw.ts index 5a1e4506..b5fe1edb 100644 --- a/src/draw.ts +++ b/src/draw.ts @@ -400,7 +400,7 @@ export async function hand(inCanvas: HTMLCanvasElement | OffscreenCanvas, result if (localOptions.drawPoints) { if (h.keypoints && h.keypoints.length > 0) { for (const pt of h.keypoints) { - ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : localOptions.color; + ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * (pt[2] || 0))}, ${127.5 - (2 * (pt[2] || 0))}, 255, 0.5)` : localOptions.color; point(ctx, pt[0], pt[1], 0, localOptions); } } diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index b647808c..2f319005 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -6,7 +6,7 @@ import { log, join } from '../util'; import * as tf from '../../dist/tfjs.esm.js'; -import type { BodyResult } from '../result'; +import type { BodyResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../env'; @@ -16,8 +16,8 @@ let model: GraphModel | null; type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] }; const keypoints: Array = []; -let box: [number, number, number, number] = [0, 0, 0, 0]; -let boxRaw: [number, number, number, number] = [0, 0, 0, 0]; +let box: Box = [0, 0, 0, 0]; +let boxRaw: Box = [0, 0, 0, 0]; let score = 0; let skipped = Number.MAX_SAFE_INTEGER; diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 20e5bed4..262eabd9 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -9,7 +9,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; import * as handpipeline from './handpipeline'; import * as fingerPose from '../fingerpose/fingerpose'; -import type { HandResult } from '../result'; +import type { HandResult, Box, Point } from '../result'; import type { Tensor, GraphModel } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../env'; @@ -39,10 +39,10 @@ export async function predict(input: Tensor, config: Config): Promise; + const keypoints = predictions[i].landmarks as unknown as Array; - let box: [number, number, number, number] = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work - let boxRaw: [number, number, number, number] = [0, 0, 0, 0]; + let box: Box = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work + let boxRaw: Box = [0, 0, 0, 0]; if (keypoints && keypoints.length > 0) { // if we have landmarks, calculate box based on landmarks for (const pt of keypoints) { if (pt[0] < box[0]) box[0] = pt[0]; diff --git a/src/handtrack/handtrack.ts b/src/handtrack/handtrack.ts index 79b1e93f..f45d4d94 100644 --- a/src/handtrack/handtrack.ts +++ b/src/handtrack/handtrack.ts @@ -8,7 +8,7 @@ import { log, join, scaleBox } from '../util'; import * as tf from '../../dist/tfjs.esm.js'; -import type { HandResult } from '../result'; +import type { HandResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../env'; @@ -29,10 +29,10 @@ let outputSize: [number, number] = [0, 0]; type HandDetectResult = { id: number, score: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], + box: Box, + boxRaw: Box, label: string, - yxBox: [number, number, number, number], + yxBox: Box, } const cache: { @@ -111,17 +111,17 @@ async function detectHands(input: Tensor, config: Config): Promise ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as Box; const boxRaw = newResult.body[i].boxRaw // update boxRaw - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as Box; const keypoints = (newResult.body[i].keypoints // update keypoints .map((keypoint, j) => ({ score: keypoint.score, @@ -56,13 +56,13 @@ export function calc(newResult: Result): Result { } else { for (let i = 0; i < newResult.hand.length; i++) { const box = (newResult.hand[i].box// update box - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as Box; const boxRaw = (newResult.hand[i].boxRaw // update boxRaw - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as Box; if (bufferedResult.hand[i].keypoints.length !== newResult.hand[i].keypoints.length) bufferedResult.hand[i].keypoints = newResult.hand[i].keypoints; // reset keypoints as previous frame did not have them const keypoints = newResult.hand[i].keypoints && newResult.hand[i].keypoints.length > 0 ? newResult.hand[i].keypoints // update landmarks .map((landmark, j) => landmark - .map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number]) + .map((coord, k) => (((bufferedFactor - 1) * (bufferedResult.hand[i].keypoints[j][k] || 1) + (coord || 0)) / bufferedFactor)) as Point) : []; const annotations = {}; if (Object.keys(bufferedResult.hand[i].annotations).length !== Object.keys(newResult.hand[i].annotations).length) bufferedResult.hand[i].annotations = newResult.hand[i].annotations; // reset annotations as previous frame did not have them @@ -83,9 +83,9 @@ export function calc(newResult: Result): Result { } else { for (let i = 0; i < newResult.face.length; i++) { const box = (newResult.face[i].box // update box - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as Box; const boxRaw = (newResult.face[i].boxRaw // update boxRaw - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as Box; const rotation: { matrix: [number, number, number, number, number, number, number, number, number], angle: { roll: number, yaw: number, pitch: number }, @@ -112,9 +112,9 @@ export function calc(newResult: Result): Result { } else { for (let i = 0; i < newResult.object.length; i++) { const box = (newResult.object[i].box // update box - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / bufferedFactor)) as Box; const boxRaw = (newResult.object[i].boxRaw // update boxRaw - .map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number]; + .map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / bufferedFactor)) as Box; bufferedResult.object[i] = { ...newResult.object[i], box, boxRaw }; // shallow clone plus updated values } } @@ -127,7 +127,7 @@ export function calc(newResult: Result): Result { } else { for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow bufferedResult.persons[i].box = (newPersons[i].box - .map((box, j) => ((bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / bufferedFactor)) as [number, number, number, number]; + .map((box, j) => ((bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / bufferedFactor)) as Box; } } } diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index 37f37c5f..e29322bd 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -6,7 +6,7 @@ import { log, join, scaleBox } from '../util'; import * as tf from '../../dist/tfjs.esm.js'; -import type { BodyResult } from '../result'; +import type { BodyResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { fakeOps } from '../tfjs/backend'; @@ -14,14 +14,11 @@ import { env } from '../env'; let model: GraphModel | null; let inputSize = 0; -const cachedBoxes: Array<[number, number, number, number]> = []; +const cachedBoxes: Array = []; type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] }; -type Body = { id: number, score: number, box: [number, number, number, number], boxRaw: [number, number, number, number], keypoints: Array } +type Body = { id: number, score: number, box: Box, boxRaw: Box, keypoints: Array } -let box: [number, number, number, number] = [0, 0, 0, 0]; -let boxRaw: [number, number, number, number] = [0, 0, 0, 0]; -let score = 0; let skipped = Number.MAX_SAFE_INTEGER; const keypoints: Array = []; @@ -43,6 +40,7 @@ export async function load(config: Config): Promise { async function parseSinglePose(res, config, image, inputBox) { const kpt = res[0][0]; keypoints.length = 0; + let score = 0; for (let id = 0; id < kpt.length; id++) { score = kpt[id][2]; if (score > config.body.minConfidence) { @@ -64,7 +62,7 @@ async function parseSinglePose(res, config, image, inputBox) { score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); const x = keypoints.map((a) => a.position[0]); const y = keypoints.map((a) => a.position[1]); - box = [ + const box: Box = [ Math.min(...x), Math.min(...y), Math.max(...x) - Math.min(...x), @@ -72,7 +70,7 @@ async function parseSinglePose(res, config, image, inputBox) { ]; const xRaw = keypoints.map((a) => a.positionRaw[0]); const yRaw = keypoints.map((a) => a.positionRaw[1]); - boxRaw = [ + const boxRaw: Box = [ Math.min(...xRaw), Math.min(...yRaw), Math.max(...xRaw) - Math.min(...xRaw), @@ -87,7 +85,7 @@ async function parseMultiPose(res, config, image, inputBox) { const bodies: Array = []; for (let id = 0; id < res[0].length; id++) { const kpt = res[0][id]; - score = Math.round(100 * kpt[51 + 4]) / 100; + const score = Math.round(100 * kpt[51 + 4]) / 100; // eslint-disable-next-line no-continue if (score < config.body.minConfidence) continue; keypoints.length = 0; @@ -106,7 +104,7 @@ async function parseMultiPose(res, config, image, inputBox) { }); } } - boxRaw = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]]; + const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]]; bodies.push({ id, score, diff --git a/src/object/centernet.ts b/src/object/centernet.ts index b3a9355c..d6ab47e4 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -7,7 +7,7 @@ import { log, join } from '../util'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; -import type { ObjectResult } from '../result'; +import type { ObjectResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../env'; @@ -60,18 +60,18 @@ async function process(res: Tensor | null, outputShape, config: Config) { detections[0][id][0] / inputSize, detections[0][id][1] / inputSize, ]; - const boxRaw = [ + const boxRaw: Box = [ x, y, detections[0][id][2] / inputSize - x, detections[0][id][3] / inputSize - y, - ] as [number, number, number, number]; - const box = [ + ]; + const box: Box = [ Math.trunc(boxRaw[0] * outputShape[0]), Math.trunc(boxRaw[1] * outputShape[1]), Math.trunc(boxRaw[2] * outputShape[0]), Math.trunc(boxRaw[3] * outputShape[1]), - ] as [number, number, number, number]; + ]; results.push({ id: i++, score, class: classVal, label, box, boxRaw }); } return results; diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index fd491f9f..bbb0c91a 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -7,7 +7,7 @@ import { log, join } from '../util'; import * as tf from '../../dist/tfjs.esm.js'; import { labels } from './labels'; -import type { ObjectResult } from '../result'; +import type { ObjectResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../env'; @@ -58,8 +58,8 @@ async function process(res, inputSize, outputShape, config) { cx + (scaleBox / strideSize * boxOffset[2]) - x, cy + (scaleBox / strideSize * boxOffset[3]) - y, ]; - let boxRaw = [x, y, w, h]; // results normalized to range 0..1 - boxRaw = boxRaw.map((a) => Math.max(0, Math.min(a, 1))); // fix out-of-bounds coords + let boxRaw: Box = [x, y, w, h]; // results normalized to range 0..1 + boxRaw = boxRaw.map((a) => Math.max(0, Math.min(a, 1))) as Box; // fix out-of-bounds coords const box = [ // results normalized to input image pixels boxRaw[0] * outputShape[0], boxRaw[1] * outputShape[1], @@ -74,8 +74,8 @@ async function process(res, inputSize, outputShape, config) { label: labels[j].label, // center: [Math.trunc(outputShape[0] * cx), Math.trunc(outputShape[1] * cy)], // centerRaw: [cx, cy], - box: (box.map((a) => Math.trunc(a))) as [number, number, number, number], - boxRaw: boxRaw as [number, number, number, number], + box: box.map((a) => Math.trunc(a)) as Box, + boxRaw, }; results.push(result); } diff --git a/src/persons.ts b/src/persons.ts index edd9755c..2529edd2 100644 --- a/src/persons.ts +++ b/src/persons.ts @@ -2,7 +2,7 @@ * Analyze detection Results and sort&combine them into per-person view */ -import type { FaceResult, BodyResult, HandResult, GestureResult, PersonResult } from './result'; +import type { FaceResult, BodyResult, HandResult, GestureResult, PersonResult, Box } from './result'; export function join(faces: Array, bodies: Array, hands: Array, gestures: Array, shape: Array | undefined): Array { let id = 0; @@ -44,7 +44,7 @@ export function join(faces: Array, bodies: Array, hands: // create new overarching box from all boxes beloning to person const x: number[] = []; const y: number[] = []; - const extractXY = (box: [number, number, number, number] | undefined) => { // extract all [x, y] coordinates from boxes [x, y, width, height] + const extractXY = (box: Box | undefined) => { // extract all [x, y] coordinates from boxes [x, y, width, height] if (box && box.length === 4) { x.push(box[0], box[0] + box[2]); y.push(box[1], box[1] + box[3]); diff --git a/src/posenet/poses.ts b/src/posenet/poses.ts index fcef4f25..2b8c75a9 100644 --- a/src/posenet/poses.ts +++ b/src/posenet/poses.ts @@ -5,6 +5,7 @@ import * as utils from './utils'; import * as kpt from './keypoints'; +import type { Box } from '../result'; const localMaximumRadius = 1; const outputStride = 16; @@ -125,7 +126,7 @@ function getInstanceScore(existingPoses, keypoints) { } export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) { - const poses: Array<{ keypoints, box: [number, number, number, number], score: number }> = []; + const poses: Array<{ keypoints, box: Box, score: number }> = []; const queue = buildPartWithScoreQueue(minConfidence, scores); // Generate at most maxDetected object instances per image in decreasing root part score order. while (poses.length < maxDetected && !queue.empty()) { diff --git a/src/result.ts b/src/result.ts index fe0280c0..75babe52 100644 --- a/src/result.ts +++ b/src/result.ts @@ -5,6 +5,9 @@ import type { Tensor } from './tfjs/types'; import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture'; +export type Box = [number, number, number, number]; +export type Point = [number, number, number?]; + /** Face results * Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models * Some values may be null if specific model is not enabled @@ -37,11 +40,11 @@ export interface FaceResult { score: number, boxScore: number, faceScore: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - mesh: Array<[number, number, number]> - meshRaw: Array<[number, number, number]> - annotations: Record>, + box: Box, + boxRaw: Box, + mesh: Array + meshRaw: Array + annotations: Record, age?: number, gender?: string, genderScore?: number, @@ -72,12 +75,12 @@ export interface FaceResult { export interface BodyResult { id: number, score: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], + box: Box, + boxRaw: Box, keypoints: Array<{ part: string, - position: [number, number, number?], - positionRaw: [number, number, number?], + position: Point, + positionRaw: Point, score: number, presence?: number, }> @@ -99,13 +102,13 @@ export interface HandResult { score: number, boxScore: number, fingerScore: number, - box: [number, number, number, number], - boxRaw: [number, number, number, number], - keypoints: Array<[number, number, number]>, + box: Box, + boxRaw: Box, + keypoints: Array, label: string, annotations: Record< 'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm', - Array<[number, number, number]> + Array >, landmarks: Record< 'index' | 'middle' | 'pinky' | 'ring' | 'thumb', @@ -130,8 +133,8 @@ export interface ObjectResult { score: number, class: number, label: string, - box: [number, number, number, number], - boxRaw: [number, number, number, number], + box: Box, + boxRaw: Box, } /** Gesture results @@ -166,8 +169,8 @@ export interface PersonResult { body: BodyResult | null, hands: { left: HandResult | null, right: HandResult | null }, gestures: Array, - box: [number, number, number, number], - boxRaw?: [number, number, number, number], + box: Box, + boxRaw?: Box, } /** diff --git a/src/util.ts b/src/util.ts index 2a8f22ba..5f845e45 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2,6 +2,8 @@ * Simple helper functions used accross codebase */ +import type { Box } from './result'; + // helper function: join two paths export function join(folder: string, file: string): string { const separator = folder.endsWith('/') ? '' : '/'; @@ -81,18 +83,18 @@ export function scaleBox(keypoints, boxScaleFact, outputSize) { Math.trunc(center[1] - diff), Math.trunc(2 * diff), Math.trunc(2 * diff), - ] as [number, number, number, number]; + ] as Box; const boxRaw = [ // work backwards box[0] / outputSize[0], box[1] / outputSize[1], box[2] / outputSize[0], box[3] / outputSize[1], - ] as [number, number, number, number]; + ] as Box; const yxBox = [ // work backwards boxRaw[1], boxRaw[0], boxRaw[3] + boxRaw[1], boxRaw[2] + boxRaw[0], - ] as [number, number, number, number]; + ] as Box; return { box, boxRaw, yxBox }; }