mirror of https://github.com/vladmandic/human
define app specific types
Vladimir Mandic
parent
a7ddea2585
commit
72aad6e812
|
|
@ -9,11 +9,11 @@
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/09/25 mandic00@live.com
|
||||
|
||||
|
||||
### **origin/main** 2021/09/25 mandic00@live.com
|
||||
### **HEAD -> main** 2021/09/27 mandic00@live.com
|
||||
|
||||
- autodetect number of bodies and hands
|
||||
- upload new samples
|
||||
- new samples gallery and major code folder restructure
|
||||
- new release
|
||||
|
||||
### **2.2.3** 2021/09/24 mandic00@live.com
|
||||
|
|
|
|||
|
|
@ -400,7 +400,7 @@ export async function hand(inCanvas: HTMLCanvasElement | OffscreenCanvas, result
|
|||
if (localOptions.drawPoints) {
|
||||
if (h.keypoints && h.keypoints.length > 0) {
|
||||
for (const pt of h.keypoints) {
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth ? `rgba(${127.5 + (2 * (pt[2] || 0))}, ${127.5 - (2 * (pt[2] || 0))}, 255, 0.5)` : localOptions.color;
|
||||
point(ctx, pt[0], pt[1], 0, localOptions);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { log, join } from '../util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { BodyResult } from '../result';
|
||||
import type { BodyResult, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../env';
|
||||
|
|
@ -16,8 +16,8 @@ let model: GraphModel | null;
|
|||
type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
|
||||
|
||||
const keypoints: Array<Keypoints> = [];
|
||||
let box: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let boxRaw: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let box: Box = [0, 0, 0, 0];
|
||||
let boxRaw: Box = [0, 0, 0, 0];
|
||||
let score = 0;
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import * as tf from '../../dist/tfjs.esm.js';
|
|||
import * as handdetector from './handdetector';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import * as fingerPose from '../fingerpose/fingerpose';
|
||||
import type { HandResult } from '../result';
|
||||
import type { HandResult, Box, Point } from '../result';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../env';
|
||||
|
|
@ -39,10 +39,10 @@ export async function predict(input: Tensor, config: Config): Promise<HandResult
|
|||
}
|
||||
}
|
||||
|
||||
const keypoints = predictions[i].landmarks as unknown as Array<[number, number, number]>;
|
||||
const keypoints = predictions[i].landmarks as unknown as Array<Point>;
|
||||
|
||||
let box: [number, number, number, number] = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work
|
||||
let boxRaw: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let box: Box = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, 0, 0]; // maximums so conditionals work
|
||||
let boxRaw: Box = [0, 0, 0, 0];
|
||||
if (keypoints && keypoints.length > 0) { // if we have landmarks, calculate box based on landmarks
|
||||
for (const pt of keypoints) {
|
||||
if (pt[0] < box[0]) box[0] = pt[0];
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
import { log, join, scaleBox } from '../util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { HandResult } from '../result';
|
||||
import type { HandResult, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../env';
|
||||
|
|
@ -29,10 +29,10 @@ let outputSize: [number, number] = [0, 0];
|
|||
type HandDetectResult = {
|
||||
id: number,
|
||||
score: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
box: Box,
|
||||
boxRaw: Box,
|
||||
label: string,
|
||||
yxBox: [number, number, number, number],
|
||||
yxBox: Box,
|
||||
}
|
||||
|
||||
const cache: {
|
||||
|
|
@ -111,17 +111,17 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
|
|||
tf.dispose(t.nms);
|
||||
for (const res of Array.from(nms)) { // generates results for each class
|
||||
const boxSlice = tf.slice(t.boxes, res, 1);
|
||||
let yxBox: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let yxBox: Box = [0, 0, 0, 0];
|
||||
if (config.hand.landmarks) { // scale box
|
||||
const detectedBox: [number, number, number, number] = await boxSlice.data();
|
||||
const detectedBox: Box = await boxSlice.data();
|
||||
const boxCenter: [number, number] = [(detectedBox[0] + detectedBox[2]) / 2, (detectedBox[1] + detectedBox[3]) / 2];
|
||||
const boxDiff: [number, number, number, number] = [+boxCenter[0] - detectedBox[0], +boxCenter[1] - detectedBox[1], -boxCenter[0] + detectedBox[2], -boxCenter[1] + detectedBox[3]];
|
||||
const boxDiff: Box = [+boxCenter[0] - detectedBox[0], +boxCenter[1] - detectedBox[1], -boxCenter[0] + detectedBox[2], -boxCenter[1] + detectedBox[3]];
|
||||
yxBox = [boxCenter[0] - boxScaleFact * boxDiff[0], boxCenter[1] - boxScaleFact * boxDiff[1], boxCenter[0] + boxScaleFact * boxDiff[2], boxCenter[1] + boxScaleFact * boxDiff[3]];
|
||||
} else { // use box as-is
|
||||
yxBox = await boxSlice.data();
|
||||
}
|
||||
const boxRaw: [number, number, number, number] = [yxBox[1], yxBox[0], yxBox[3] - yxBox[1], yxBox[2] - yxBox[0]];
|
||||
const box: [number, number, number, number] = [Math.trunc(boxRaw[0] * outputSize[0]), Math.trunc(boxRaw[1] * outputSize[1]), Math.trunc(boxRaw[2] * outputSize[0]), Math.trunc(boxRaw[3] * outputSize[1])];
|
||||
const boxRaw: Box = [yxBox[1], yxBox[0], yxBox[3] - yxBox[1], yxBox[2] - yxBox[0]];
|
||||
const box: Box = [Math.trunc(boxRaw[0] * outputSize[0]), Math.trunc(boxRaw[1] * outputSize[1]), Math.trunc(boxRaw[2] * outputSize[0]), Math.trunc(boxRaw[3] * outputSize[1])];
|
||||
tf.dispose(boxSlice);
|
||||
const scoreSlice = tf.slice(classScores[i], res, 1);
|
||||
const score = (await scoreSlice.data())[0];
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ export * from './config';
|
|||
export * from './result';
|
||||
export type { DrawOptions } from './draw';
|
||||
export { env, Env } from './env';
|
||||
export { Box, Point } from './result';
|
||||
export { Models } from './models';
|
||||
|
||||
/** Defines all possible input types for **Human** detection
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Results interpolation for smoothening of video detection results inbetween detected frames
|
||||
*/
|
||||
|
||||
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult } from './result';
|
||||
import type { Result, FaceResult, BodyResult, HandResult, ObjectResult, GestureResult, PersonResult, Box, Point } from './result';
|
||||
|
||||
const bufferedResult: Result = { face: [], body: [], hand: [], gesture: [], object: [], persons: [], performance: {}, timestamp: 0 };
|
||||
|
||||
|
|
@ -30,9 +30,9 @@ export function calc(newResult: Result): Result {
|
|||
} else {
|
||||
for (let i = 0; i < newResult.body.length; i++) {
|
||||
const box = newResult.body[i].box // update box
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].box[j] + b) / bufferedFactor) as Box;
|
||||
const boxRaw = newResult.body[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.body[i].boxRaw[j] + b) / bufferedFactor) as Box;
|
||||
const keypoints = (newResult.body[i].keypoints // update keypoints
|
||||
.map((keypoint, j) => ({
|
||||
score: keypoint.score,
|
||||
|
|
@ -56,13 +56,13 @@ export function calc(newResult: Result): Result {
|
|||
} else {
|
||||
for (let i = 0; i < newResult.hand.length; i++) {
|
||||
const box = (newResult.hand[i].box// update box
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].box[j] + b) / bufferedFactor)) as Box;
|
||||
const boxRaw = (newResult.hand[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.hand[i].boxRaw[j] + b) / bufferedFactor)) as Box;
|
||||
if (bufferedResult.hand[i].keypoints.length !== newResult.hand[i].keypoints.length) bufferedResult.hand[i].keypoints = newResult.hand[i].keypoints; // reset keypoints as previous frame did not have them
|
||||
const keypoints = newResult.hand[i].keypoints && newResult.hand[i].keypoints.length > 0 ? newResult.hand[i].keypoints // update landmarks
|
||||
.map((landmark, j) => landmark
|
||||
.map((coord, k) => (((bufferedFactor - 1) * bufferedResult.hand[i].keypoints[j][k] + coord) / bufferedFactor)) as [number, number, number])
|
||||
.map((coord, k) => (((bufferedFactor - 1) * (bufferedResult.hand[i].keypoints[j][k] || 1) + (coord || 0)) / bufferedFactor)) as Point)
|
||||
: [];
|
||||
const annotations = {};
|
||||
if (Object.keys(bufferedResult.hand[i].annotations).length !== Object.keys(newResult.hand[i].annotations).length) bufferedResult.hand[i].annotations = newResult.hand[i].annotations; // reset annotations as previous frame did not have them
|
||||
|
|
@ -83,9 +83,9 @@ export function calc(newResult: Result): Result {
|
|||
} else {
|
||||
for (let i = 0; i < newResult.face.length; i++) {
|
||||
const box = (newResult.face[i].box // update box
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].box[j] + b) / bufferedFactor)) as Box;
|
||||
const boxRaw = (newResult.face[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.face[i].boxRaw[j] + b) / bufferedFactor)) as Box;
|
||||
const rotation: {
|
||||
matrix: [number, number, number, number, number, number, number, number, number],
|
||||
angle: { roll: number, yaw: number, pitch: number },
|
||||
|
|
@ -112,9 +112,9 @@ export function calc(newResult: Result): Result {
|
|||
} else {
|
||||
for (let i = 0; i < newResult.object.length; i++) {
|
||||
const box = (newResult.object[i].box // update box
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].box[j] + b) / bufferedFactor)) as Box;
|
||||
const boxRaw = (newResult.object[i].boxRaw // update boxRaw
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((b, j) => ((bufferedFactor - 1) * bufferedResult.object[i].boxRaw[j] + b) / bufferedFactor)) as Box;
|
||||
bufferedResult.object[i] = { ...newResult.object[i], box, boxRaw }; // shallow clone plus updated values
|
||||
}
|
||||
}
|
||||
|
|
@ -127,7 +127,7 @@ export function calc(newResult: Result): Result {
|
|||
} else {
|
||||
for (let i = 0; i < newPersons.length; i++) { // update person box, we don't update the rest as it's updated as reference anyhow
|
||||
bufferedResult.persons[i].box = (newPersons[i].box
|
||||
.map((box, j) => ((bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / bufferedFactor)) as [number, number, number, number];
|
||||
.map((box, j) => ((bufferedFactor - 1) * bufferedResult.persons[i].box[j] + box) / bufferedFactor)) as Box;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { log, join, scaleBox } from '../util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { BodyResult } from '../result';
|
||||
import type { BodyResult, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { fakeOps } from '../tfjs/backend';
|
||||
|
|
@ -14,14 +14,11 @@ import { env } from '../env';
|
|||
|
||||
let model: GraphModel | null;
|
||||
let inputSize = 0;
|
||||
const cachedBoxes: Array<[number, number, number, number]> = [];
|
||||
const cachedBoxes: Array<Box> = [];
|
||||
|
||||
type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
|
||||
type Body = { id: number, score: number, box: [number, number, number, number], boxRaw: [number, number, number, number], keypoints: Array<Keypoints> }
|
||||
type Body = { id: number, score: number, box: Box, boxRaw: Box, keypoints: Array<Keypoints> }
|
||||
|
||||
let box: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let boxRaw: [number, number, number, number] = [0, 0, 0, 0];
|
||||
let score = 0;
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
const keypoints: Array<Keypoints> = [];
|
||||
|
||||
|
|
@ -43,6 +40,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
async function parseSinglePose(res, config, image, inputBox) {
|
||||
const kpt = res[0][0];
|
||||
keypoints.length = 0;
|
||||
let score = 0;
|
||||
for (let id = 0; id < kpt.length; id++) {
|
||||
score = kpt[id][2];
|
||||
if (score > config.body.minConfidence) {
|
||||
|
|
@ -64,7 +62,7 @@ async function parseSinglePose(res, config, image, inputBox) {
|
|||
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
|
||||
const x = keypoints.map((a) => a.position[0]);
|
||||
const y = keypoints.map((a) => a.position[1]);
|
||||
box = [
|
||||
const box: Box = [
|
||||
Math.min(...x),
|
||||
Math.min(...y),
|
||||
Math.max(...x) - Math.min(...x),
|
||||
|
|
@ -72,7 +70,7 @@ async function parseSinglePose(res, config, image, inputBox) {
|
|||
];
|
||||
const xRaw = keypoints.map((a) => a.positionRaw[0]);
|
||||
const yRaw = keypoints.map((a) => a.positionRaw[1]);
|
||||
boxRaw = [
|
||||
const boxRaw: Box = [
|
||||
Math.min(...xRaw),
|
||||
Math.min(...yRaw),
|
||||
Math.max(...xRaw) - Math.min(...xRaw),
|
||||
|
|
@ -87,7 +85,7 @@ async function parseMultiPose(res, config, image, inputBox) {
|
|||
const bodies: Array<Body> = [];
|
||||
for (let id = 0; id < res[0].length; id++) {
|
||||
const kpt = res[0][id];
|
||||
score = Math.round(100 * kpt[51 + 4]) / 100;
|
||||
const score = Math.round(100 * kpt[51 + 4]) / 100;
|
||||
// eslint-disable-next-line no-continue
|
||||
if (score < config.body.minConfidence) continue;
|
||||
keypoints.length = 0;
|
||||
|
|
@ -106,7 +104,7 @@ async function parseMultiPose(res, config, image, inputBox) {
|
|||
});
|
||||
}
|
||||
}
|
||||
boxRaw = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
|
||||
const boxRaw: Box = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]];
|
||||
bodies.push({
|
||||
id,
|
||||
score,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
import { log, join } from '../util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import type { ObjectResult } from '../result';
|
||||
import type { ObjectResult, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../env';
|
||||
|
|
@ -60,18 +60,18 @@ async function process(res: Tensor | null, outputShape, config: Config) {
|
|||
detections[0][id][0] / inputSize,
|
||||
detections[0][id][1] / inputSize,
|
||||
];
|
||||
const boxRaw = [
|
||||
const boxRaw: Box = [
|
||||
x,
|
||||
y,
|
||||
detections[0][id][2] / inputSize - x,
|
||||
detections[0][id][3] / inputSize - y,
|
||||
] as [number, number, number, number];
|
||||
const box = [
|
||||
];
|
||||
const box: Box = [
|
||||
Math.trunc(boxRaw[0] * outputShape[0]),
|
||||
Math.trunc(boxRaw[1] * outputShape[1]),
|
||||
Math.trunc(boxRaw[2] * outputShape[0]),
|
||||
Math.trunc(boxRaw[3] * outputShape[1]),
|
||||
] as [number, number, number, number];
|
||||
];
|
||||
results.push({ id: i++, score, class: classVal, label, box, boxRaw });
|
||||
}
|
||||
return results;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
import { log, join } from '../util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import type { ObjectResult } from '../result';
|
||||
import type { ObjectResult, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../env';
|
||||
|
|
@ -58,8 +58,8 @@ async function process(res, inputSize, outputShape, config) {
|
|||
cx + (scaleBox / strideSize * boxOffset[2]) - x,
|
||||
cy + (scaleBox / strideSize * boxOffset[3]) - y,
|
||||
];
|
||||
let boxRaw = [x, y, w, h]; // results normalized to range 0..1
|
||||
boxRaw = boxRaw.map((a) => Math.max(0, Math.min(a, 1))); // fix out-of-bounds coords
|
||||
let boxRaw: Box = [x, y, w, h]; // results normalized to range 0..1
|
||||
boxRaw = boxRaw.map((a) => Math.max(0, Math.min(a, 1))) as Box; // fix out-of-bounds coords
|
||||
const box = [ // results normalized to input image pixels
|
||||
boxRaw[0] * outputShape[0],
|
||||
boxRaw[1] * outputShape[1],
|
||||
|
|
@ -74,8 +74,8 @@ async function process(res, inputSize, outputShape, config) {
|
|||
label: labels[j].label,
|
||||
// center: [Math.trunc(outputShape[0] * cx), Math.trunc(outputShape[1] * cy)],
|
||||
// centerRaw: [cx, cy],
|
||||
box: (box.map((a) => Math.trunc(a))) as [number, number, number, number],
|
||||
boxRaw: boxRaw as [number, number, number, number],
|
||||
box: box.map((a) => Math.trunc(a)) as Box,
|
||||
boxRaw,
|
||||
};
|
||||
results.push(result);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Analyze detection Results and sort&combine them into per-person view
|
||||
*/
|
||||
|
||||
import type { FaceResult, BodyResult, HandResult, GestureResult, PersonResult } from './result';
|
||||
import type { FaceResult, BodyResult, HandResult, GestureResult, PersonResult, Box } from './result';
|
||||
|
||||
export function join(faces: Array<FaceResult>, bodies: Array<BodyResult>, hands: Array<HandResult>, gestures: Array<GestureResult>, shape: Array<number> | undefined): Array<PersonResult> {
|
||||
let id = 0;
|
||||
|
|
@ -44,7 +44,7 @@ export function join(faces: Array<FaceResult>, bodies: Array<BodyResult>, hands:
|
|||
// create new overarching box from all boxes beloning to person
|
||||
const x: number[] = [];
|
||||
const y: number[] = [];
|
||||
const extractXY = (box: [number, number, number, number] | undefined) => { // extract all [x, y] coordinates from boxes [x, y, width, height]
|
||||
const extractXY = (box: Box | undefined) => { // extract all [x, y] coordinates from boxes [x, y, width, height]
|
||||
if (box && box.length === 4) {
|
||||
x.push(box[0], box[0] + box[2]);
|
||||
y.push(box[1], box[1] + box[3]);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
import * as utils from './utils';
|
||||
import * as kpt from './keypoints';
|
||||
import type { Box } from '../result';
|
||||
|
||||
const localMaximumRadius = 1;
|
||||
const outputStride = 16;
|
||||
|
|
@ -125,7 +126,7 @@ function getInstanceScore(existingPoses, keypoints) {
|
|||
}
|
||||
|
||||
export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) {
|
||||
const poses: Array<{ keypoints, box: [number, number, number, number], score: number }> = [];
|
||||
const poses: Array<{ keypoints, box: Box, score: number }> = [];
|
||||
const queue = buildPartWithScoreQueue(minConfidence, scores);
|
||||
// Generate at most maxDetected object instances per image in decreasing root part score order.
|
||||
while (poses.length < maxDetected && !queue.empty()) {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@
|
|||
import type { Tensor } from './tfjs/types';
|
||||
import type { FaceGesture, BodyGesture, HandGesture, IrisGesture } from './gesture/gesture';
|
||||
|
||||
export type Box = [number, number, number, number];
|
||||
export type Point = [number, number, number?];
|
||||
|
||||
/** Face results
|
||||
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
|
||||
* Some values may be null if specific model is not enabled
|
||||
|
|
@ -37,11 +40,11 @@ export interface FaceResult {
|
|||
score: number,
|
||||
boxScore: number,
|
||||
faceScore: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
mesh: Array<[number, number, number]>
|
||||
meshRaw: Array<[number, number, number]>
|
||||
annotations: Record<string, Array<[number, number, number]>>,
|
||||
box: Box,
|
||||
boxRaw: Box,
|
||||
mesh: Array<Point>
|
||||
meshRaw: Array<Point>
|
||||
annotations: Record<string, Point[]>,
|
||||
age?: number,
|
||||
gender?: string,
|
||||
genderScore?: number,
|
||||
|
|
@ -72,12 +75,12 @@ export interface FaceResult {
|
|||
export interface BodyResult {
|
||||
id: number,
|
||||
score: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
box: Box,
|
||||
boxRaw: Box,
|
||||
keypoints: Array<{
|
||||
part: string,
|
||||
position: [number, number, number?],
|
||||
positionRaw: [number, number, number?],
|
||||
position: Point,
|
||||
positionRaw: Point,
|
||||
score: number,
|
||||
presence?: number,
|
||||
}>
|
||||
|
|
@ -99,13 +102,13 @@ export interface HandResult {
|
|||
score: number,
|
||||
boxScore: number,
|
||||
fingerScore: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
keypoints: Array<[number, number, number]>,
|
||||
box: Box,
|
||||
boxRaw: Box,
|
||||
keypoints: Array<Point>,
|
||||
label: string,
|
||||
annotations: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb' | 'palm',
|
||||
Array<[number, number, number]>
|
||||
Array<Point>
|
||||
>,
|
||||
landmarks: Record<
|
||||
'index' | 'middle' | 'pinky' | 'ring' | 'thumb',
|
||||
|
|
@ -130,8 +133,8 @@ export interface ObjectResult {
|
|||
score: number,
|
||||
class: number,
|
||||
label: string,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
box: Box,
|
||||
boxRaw: Box,
|
||||
}
|
||||
|
||||
/** Gesture results
|
||||
|
|
@ -166,8 +169,8 @@ export interface PersonResult {
|
|||
body: BodyResult | null,
|
||||
hands: { left: HandResult | null, right: HandResult | null },
|
||||
gestures: Array<GestureResult>,
|
||||
box: [number, number, number, number],
|
||||
boxRaw?: [number, number, number, number],
|
||||
box: Box,
|
||||
boxRaw?: Box,
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
* Simple helper functions used accross codebase
|
||||
*/
|
||||
|
||||
import type { Box } from './result';
|
||||
|
||||
// helper function: join two paths
|
||||
export function join(folder: string, file: string): string {
|
||||
const separator = folder.endsWith('/') ? '' : '/';
|
||||
|
|
@ -81,18 +83,18 @@ export function scaleBox(keypoints, boxScaleFact, outputSize) {
|
|||
Math.trunc(center[1] - diff),
|
||||
Math.trunc(2 * diff),
|
||||
Math.trunc(2 * diff),
|
||||
] as [number, number, number, number];
|
||||
] as Box;
|
||||
const boxRaw = [ // work backwards
|
||||
box[0] / outputSize[0],
|
||||
box[1] / outputSize[1],
|
||||
box[2] / outputSize[0],
|
||||
box[3] / outputSize[1],
|
||||
] as [number, number, number, number];
|
||||
] as Box;
|
||||
const yxBox = [ // work backwards
|
||||
boxRaw[1],
|
||||
boxRaw[0],
|
||||
boxRaw[3] + boxRaw[1],
|
||||
boxRaw[2] + boxRaw[0],
|
||||
] as [number, number, number, number];
|
||||
] as Box;
|
||||
return { box, boxRaw, yxBox };
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue