mirror of https://github.com/vladmandic/human
restructure results strong typing
Vladimir Mandic
parent
618ef6f7fa
commit
714d95f6ed
|
|
@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/05/21 mandic00@live.com
|
||||
|
||||
|
||||
### **1.9.1** 2021/05/21 mandic00@live.com
|
||||
|
||||
|
||||
### **origin/main** 2021/05/20 mandic00@live.com
|
||||
|
||||
- caching improvements
|
||||
- sanitize server input
|
||||
- remove nanodet weights from default distribution
|
||||
- add experimental mb3-centernet object detection
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ let human;
|
|||
|
||||
const userConfig = {
|
||||
warmup: 'none',
|
||||
/*
|
||||
backend: 'webgl',
|
||||
async: false,
|
||||
cacheSensitivity: 0,
|
||||
|
|
@ -27,10 +26,9 @@ const userConfig = {
|
|||
},
|
||||
hand: { enabled: false },
|
||||
gesture: { enabled: false },
|
||||
body: { enabled: false, modelPath: 'posenet.json' },
|
||||
body: { enabled: true, modelPath: 'posenet.json' },
|
||||
// body: { enabled: true, modelPath: 'blazepose.json' },
|
||||
object: { enabled: false },
|
||||
*/
|
||||
};
|
||||
|
||||
// ui options
|
||||
|
|
@ -46,6 +44,7 @@ const ui = {
|
|||
maxFPSframes: 10, // keep fps history for how many frames
|
||||
modelsPreload: true, // preload human models on startup
|
||||
modelsWarmup: true, // warmup human models on startup
|
||||
buffered: true, // should output be buffered between frames
|
||||
|
||||
// internal variables
|
||||
busy: false, // internal camera busy flag
|
||||
|
|
@ -54,7 +53,6 @@ const ui = {
|
|||
camera: {}, // internal, holds details of webcam details
|
||||
detectFPS: [], // internal, holds fps values for detection performance
|
||||
drawFPS: [], // internal, holds fps values for draw performance
|
||||
buffered: false, // should output be buffered between frames
|
||||
drawWarmup: false, // debug only, should warmup image processing be displayed on startup
|
||||
drawThread: null, // internl, perform draw operations in a separate thread
|
||||
detectThread: null, // internl, perform detect operations in a separate thread
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { TRI468 as triangulation } from '../blazeface/coords';
|
||||
import { mergeDeep } from '../helpers';
|
||||
import type { Result, Face, Body, Hand, Item, Gesture } from '../result';
|
||||
|
||||
/**
|
||||
* Draw Options
|
||||
|
|
@ -59,7 +60,7 @@ export const options: DrawOptions = {
|
|||
fillPolygons: <Boolean>false,
|
||||
useDepth: <Boolean>true,
|
||||
useCurves: <Boolean>false,
|
||||
bufferedOutput: <Boolean>false,
|
||||
bufferedOutput: <Boolean>true,
|
||||
useRawBoxes: <Boolean>false,
|
||||
calculateHandBox: <Boolean>true,
|
||||
};
|
||||
|
|
@ -93,14 +94,14 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
ctx.stroke();
|
||||
}
|
||||
|
||||
function lines(ctx, points: number[] = [], localOptions) {
|
||||
function lines(ctx, points: [number, number, number][] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], parseInt(pt[1]));
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
if (localOptions.fillPolygons) {
|
||||
|
|
@ -109,7 +110,7 @@ function lines(ctx, points: number[] = [], localOptions) {
|
|||
}
|
||||
}
|
||||
|
||||
function curves(ctx, points: number[] = [], localOptions) {
|
||||
function curves(ctx, points: [number, number, number][] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
|
@ -129,7 +130,7 @@ function curves(ctx, points: number[] = [], localOptions) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
export async function gesture(inCanvas: HTMLCanvasElement, result: Array<Gesture>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
@ -156,7 +157,7 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array<any>, d
|
|||
}
|
||||
}
|
||||
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
@ -211,24 +212,24 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
lines(ctx, points, localOptions);
|
||||
}
|
||||
// iris: array[center, left, top, right, bottom]
|
||||
if (f.annotations && f.annotations.leftEyeIris) {
|
||||
if (f.annotations && f.annotations['leftEyeIris']) {
|
||||
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
|
||||
ctx.beginPath();
|
||||
const sizeX = Math.abs(f.annotations.leftEyeIris[3][0] - f.annotations.leftEyeIris[1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations.leftEyeIris[4][1] - f.annotations.leftEyeIris[2][1]) / 2;
|
||||
ctx.ellipse(f.annotations.leftEyeIris[0][0], f.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
const sizeX = Math.abs(f.annotations['leftEyeIris'][3][0] - f.annotations['leftEyeIris'][1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations['leftEyeIris'][4][1] - f.annotations['leftEyeIris'][2][1]) / 2;
|
||||
ctx.ellipse(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
ctx.stroke();
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
if (f.annotations && f.annotations.rightEyeIris) {
|
||||
if (f.annotations && f.annotations['rightEyeIris']) {
|
||||
ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color;
|
||||
ctx.beginPath();
|
||||
const sizeX = Math.abs(f.annotations.rightEyeIris[3][0] - f.annotations.rightEyeIris[1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations.rightEyeIris[4][1] - f.annotations.rightEyeIris[2][1]) / 2;
|
||||
ctx.ellipse(f.annotations.rightEyeIris[0][0], f.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
const sizeX = Math.abs(f.annotations['rightEyeIris'][3][0] - f.annotations['rightEyeIris'][1][0]) / 2;
|
||||
const sizeY = Math.abs(f.annotations['rightEyeIris'][4][1] - f.annotations['rightEyeIris'][2][1]) / 2;
|
||||
ctx.ellipse(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
|
||||
ctx.stroke();
|
||||
if (localOptions.fillPolygons) {
|
||||
ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color;
|
||||
|
|
@ -241,7 +242,7 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
}
|
||||
|
||||
const lastDrawnPose:any[] = [];
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
@ -249,20 +250,22 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
if (!ctx) return;
|
||||
ctx.lineJoin = 'round';
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
// result[i].keypoints = result[i].keypoints.filter((a) => a.score > 0.5);
|
||||
if (!lastDrawnPose[i] && localOptions.bufferedOutput) lastDrawnPose[i] = { ...result[i] };
|
||||
ctx.strokeStyle = localOptions.color;
|
||||
ctx.fillStyle = localOptions.color;
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
ctx.font = localOptions.font;
|
||||
if (localOptions.drawBoxes) {
|
||||
if (localOptions.drawBoxes && result[i].box && result[i].box?.length === 4) {
|
||||
// @ts-ignore box may not exist
|
||||
rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions);
|
||||
if (localOptions.drawLabels) {
|
||||
if (localOptions.shadowColor && localOptions.shadowColor !== '') {
|
||||
ctx.fillStyle = localOptions.shadowColor;
|
||||
// @ts-ignore box may not exist
|
||||
ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]);
|
||||
}
|
||||
ctx.fillStyle = localOptions.labelColor;
|
||||
// @ts-ignore box may not exist
|
||||
ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]);
|
||||
}
|
||||
}
|
||||
|
|
@ -361,7 +364,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
}
|
||||
}
|
||||
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
export async function hand(inCanvas: HTMLCanvasElement, result: Array<Hand>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
@ -415,12 +418,12 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4);
|
||||
};
|
||||
ctx.font = localOptions.font;
|
||||
addHandLabel(h.annotations.indexFinger, 'index');
|
||||
addHandLabel(h.annotations.middleFinger, 'middle');
|
||||
addHandLabel(h.annotations.ringFinger, 'ring');
|
||||
addHandLabel(h.annotations.pinky, 'pinky');
|
||||
addHandLabel(h.annotations.thumb, 'thumb');
|
||||
addHandLabel(h.annotations.palmBase, 'palm');
|
||||
addHandLabel(h.annotations['indexFinger'], 'index');
|
||||
addHandLabel(h.annotations['middleFinger'], 'middle');
|
||||
addHandLabel(h.annotations['ringFinger'], 'ring');
|
||||
addHandLabel(h.annotations['pinky'], 'pinky');
|
||||
addHandLabel(h.annotations['thumb'], 'thumb');
|
||||
addHandLabel(h.annotations['palmBase'], 'palm');
|
||||
}
|
||||
if (localOptions.drawPolygons) {
|
||||
const addHandLine = (part) => {
|
||||
|
|
@ -434,17 +437,17 @@ export async function hand(inCanvas: HTMLCanvasElement, result: Array<any>, draw
|
|||
}
|
||||
};
|
||||
ctx.lineWidth = localOptions.lineWidth;
|
||||
addHandLine(h.annotations.indexFinger);
|
||||
addHandLine(h.annotations.middleFinger);
|
||||
addHandLine(h.annotations.ringFinger);
|
||||
addHandLine(h.annotations.pinky);
|
||||
addHandLine(h.annotations.thumb);
|
||||
addHandLine(h.annotations['indexFinger']);
|
||||
addHandLine(h.annotations['middleFinger']);
|
||||
addHandLine(h.annotations['ringFinger']);
|
||||
addHandLine(h.annotations['pinky']);
|
||||
addHandLine(h.annotations['thumb']);
|
||||
// addPart(h.annotations.palmBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<any>, drawOptions?: DrawOptions) {
|
||||
export async function object(inCanvas: HTMLCanvasElement, result: Array<Item>, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
@ -479,7 +482,7 @@ export async function canvas(inCanvas: HTMLCanvasElement, outCanvas: HTMLCanvasE
|
|||
outCtx?.drawImage(inCanvas, 0, 0);
|
||||
}
|
||||
|
||||
export async function all(inCanvas: HTMLCanvasElement, result:any, drawOptions?: DrawOptions) {
|
||||
export async function all(inCanvas: HTMLCanvasElement, result: Result, drawOptions?: DrawOptions) {
|
||||
const localOptions = mergeDeep(options, drawOptions);
|
||||
if (!result || !inCanvas) return;
|
||||
if (!(inCanvas instanceof HTMLCanvasElement)) return;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { Body } from '../result';
|
||||
|
||||
let model;
|
||||
let keypoints: Array<any> = [];
|
||||
|
|
@ -37,8 +38,7 @@ function max2d(inputs, minScore) {
|
|||
});
|
||||
}
|
||||
|
||||
export async function predict(image, config) {
|
||||
if (!model) return null;
|
||||
export async function predict(image, config): Promise<Body[]> {
|
||||
if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
skipped++;
|
||||
return keypoints;
|
||||
|
|
@ -87,6 +87,6 @@ export async function predict(image, config) {
|
|||
keypoints = parts;
|
||||
}
|
||||
const score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
|
||||
resolve([{ score, keypoints }]);
|
||||
resolve([{ id: 0, score, keypoints }]);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
27
src/face.ts
27
src/face.ts
|
|
@ -1,10 +1,8 @@
|
|||
import { log, now } from './helpers';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as facemesh from './blazeface/facemesh';
|
||||
import * as emotion from './emotion/emotion';
|
||||
import * as faceres from './faceres/faceres';
|
||||
|
||||
type Tensor = typeof tf.Tensor;
|
||||
import { Face } from './result';
|
||||
|
||||
const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: number, roll: number }, matrix: [number, number, number, number, number, number, number, number, number] } => {
|
||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||
|
|
@ -107,27 +105,7 @@ export const detectFace = async (parent, input): Promise<any> => {
|
|||
let emotionRes;
|
||||
let embeddingRes;
|
||||
let descRes;
|
||||
const faceRes: Array<{
|
||||
confidence: number,
|
||||
boxConfidence: number,
|
||||
faceConfidence: number,
|
||||
box: [number, number, number, number],
|
||||
mesh: Array<[number, number, number]>
|
||||
meshRaw: Array<[number, number, number]>
|
||||
boxRaw: [number, number, number, number],
|
||||
annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
|
||||
age: number,
|
||||
gender: string,
|
||||
genderConfidence: number,
|
||||
emotion: string,
|
||||
embedding: number[],
|
||||
iris: number,
|
||||
rotation: {
|
||||
angle: { pitch: number, yaw: number, roll: number },
|
||||
matrix: [number, number, number, number, number, number, number, number, number]
|
||||
},
|
||||
tensor: Tensor,
|
||||
}> = [];
|
||||
const faceRes: Array<Face> = [];
|
||||
parent.state = 'run:face';
|
||||
timeStamp = now();
|
||||
const faces = await facemesh.predict(input, parent.config);
|
||||
|
|
@ -189,6 +167,7 @@ export const detectFace = async (parent, input): Promise<any> => {
|
|||
|
||||
// combine results
|
||||
faceRes.push({
|
||||
id: i,
|
||||
...faces[i],
|
||||
age: descRes.age,
|
||||
gender: descRes.gender,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
export const body = (res) => {
|
||||
import { Gesture } from '../result';
|
||||
|
||||
export const body = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ body: number, gesture: string }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
|
@ -18,7 +20,7 @@ export const body = (res) => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const face = (res) => {
|
||||
export const face = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ face: number, gesture: string }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
|
@ -39,7 +41,7 @@ export const face = (res) => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const iris = (res) => {
|
||||
export const iris = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ iris: number, gesture: string }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
|
@ -77,7 +79,7 @@ export const iris = (res) => {
|
|||
return gestures;
|
||||
};
|
||||
|
||||
export const hand = (res) => {
|
||||
export const hand = (res): Gesture[] => {
|
||||
if (!res) return [];
|
||||
const gestures: Array<{ hand: number, gesture: string }> = [];
|
||||
for (let i = 0; i < res.length; i++) {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { log, join } from '../helpers';
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as handdetector from './handdetector';
|
||||
import * as handpipeline from './handpipeline';
|
||||
import { Hand } from '../result';
|
||||
|
||||
const meshAnnotations = {
|
||||
thumb: [1, 2, 3, 4],
|
||||
|
|
@ -16,30 +17,30 @@ let handDetectorModel;
|
|||
let handPoseModel;
|
||||
let handPipeline;
|
||||
|
||||
export async function predict(input, config) {
|
||||
export async function predict(input, config): Promise<Hand[]> {
|
||||
const predictions = await handPipeline.estimateHands(input, config);
|
||||
if (!predictions) return [];
|
||||
const hands: Array<{ confidence: number, box: any, boxRaw: any, landmarks: any, annotations: any }> = [];
|
||||
for (const prediction of predictions) {
|
||||
const hands: Array<Hand> = [];
|
||||
for (let i = 0; i < predictions.length; i++) {
|
||||
const annotations = {};
|
||||
if (prediction.landmarks) {
|
||||
if (predictions[i].landmarks) {
|
||||
for (const key of Object.keys(meshAnnotations)) {
|
||||
annotations[key] = meshAnnotations[key].map((index) => prediction.landmarks[index]);
|
||||
annotations[key] = meshAnnotations[key].map((index) => predictions[i].landmarks[index]);
|
||||
}
|
||||
}
|
||||
const box = prediction.box ? [
|
||||
Math.max(0, prediction.box.topLeft[0]),
|
||||
Math.max(0, prediction.box.topLeft[1]),
|
||||
Math.min(input.shape[2], prediction.box.bottomRight[0]) - Math.max(0, prediction.box.topLeft[0]),
|
||||
Math.min(input.shape[1], prediction.box.bottomRight[1]) - Math.max(0, prediction.box.topLeft[1]),
|
||||
] : [];
|
||||
const boxRaw = [
|
||||
(prediction.box.topLeft[0]) / input.shape[2],
|
||||
(prediction.box.topLeft[1]) / input.shape[1],
|
||||
(prediction.box.bottomRight[0] - prediction.box.topLeft[0]) / input.shape[2],
|
||||
(prediction.box.bottomRight[1] - prediction.box.topLeft[1]) / input.shape[1],
|
||||
const box: [number, number, number, number] = predictions[i].box ? [
|
||||
Math.max(0, predictions[i].box.topLeft[0]),
|
||||
Math.max(0, predictions[i].box.topLeft[1]),
|
||||
Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0]),
|
||||
Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1]),
|
||||
] : [0, 0, 0, 0];
|
||||
const boxRaw: [number, number, number, number] = [
|
||||
(predictions[i].box.topLeft[0]) / input.shape[2],
|
||||
(predictions[i].box.topLeft[1]) / input.shape[1],
|
||||
(predictions[i].box.bottomRight[0] - predictions[i].box.topLeft[0]) / input.shape[2],
|
||||
(predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / input.shape[1],
|
||||
];
|
||||
hands.push({ confidence: Math.round(100 * prediction.confidence) / 100, box, boxRaw, landmarks: prediction.landmarks, annotations });
|
||||
hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, landmarks: predictions[i].landmarks, annotations });
|
||||
}
|
||||
return hands;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import * as app from '../package.json';
|
|||
export type Tensor = typeof tf.Tensor;
|
||||
|
||||
export type { Config } from './config';
|
||||
export type { Result } from './result';
|
||||
export type { Result, Face, Hand, Body, Item, Gesture } from './result';
|
||||
export type { DrawOptions } from './draw/draw';
|
||||
|
||||
/** Defines all possible input types for **Human** detection */
|
||||
|
|
@ -530,7 +530,7 @@ export class Human {
|
|||
|
||||
this.perf.total = Math.trunc(now() - timeStart);
|
||||
this.state = 'idle';
|
||||
const result = {
|
||||
const res = {
|
||||
face: faceRes,
|
||||
body: bodyRes,
|
||||
hand: handRes,
|
||||
|
|
@ -540,7 +540,7 @@ export class Human {
|
|||
canvas: process.canvas,
|
||||
};
|
||||
// log('Result:', result);
|
||||
resolve(result);
|
||||
resolve(res);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import { Item } from '../result';
|
||||
|
||||
let model;
|
||||
let last: Array<{}> = [];
|
||||
let last: Item[] = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
export async function load(config) {
|
||||
|
|
@ -58,8 +59,7 @@ async function process(res, inputSize, outputShape, config) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(image, config) {
|
||||
if (!model) return null;
|
||||
export async function predict(image, config): Promise<Item[]> {
|
||||
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
|
||||
skipped++;
|
||||
return last;
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { log, join } from '../helpers';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { labels } from './labels';
|
||||
import { Item } from '../result';
|
||||
|
||||
let model;
|
||||
let last: Array<{}> = [];
|
||||
let last: Array<Item> = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
const scaleBox = 2.5; // increase box size
|
||||
|
|
@ -95,8 +96,7 @@ async function process(res, inputSize, outputShape, config) {
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(image, config) {
|
||||
if (!model) return null;
|
||||
export async function predict(image, config): Promise<Item[]> {
|
||||
if ((skipped < config.object.skipFrames) && config.skipFrame && (last.length > 0)) {
|
||||
skipped++;
|
||||
return last;
|
||||
|
|
|
|||
|
|
@ -30,8 +30,11 @@ export function getBoundingBox(keypoints) {
|
|||
}
|
||||
|
||||
export function scalePoses(poses, [height, width], [inputResolutionHeight, inputResolutionWidth]) {
|
||||
const scalePose = (pose, scaleY, scaleX) => ({
|
||||
const scaleY = height / inputResolutionHeight;
|
||||
const scaleX = width / inputResolutionWidth;
|
||||
const scalePose = (pose) => ({
|
||||
score: pose.score,
|
||||
bowRaw: [pose.box[0] / inputResolutionWidth, pose.box[1] / inputResolutionHeight, pose.box[2] / inputResolutionWidth, pose.box[3] / inputResolutionHeight],
|
||||
box: [Math.trunc(pose.box[0] * scaleX), Math.trunc(pose.box[1] * scaleY), Math.trunc(pose.box[2] * scaleX), Math.trunc(pose.box[3] * scaleY)],
|
||||
keypoints: pose.keypoints.map(({ score, part, position }) => ({
|
||||
score,
|
||||
|
|
@ -39,7 +42,7 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input
|
|||
position: { x: Math.trunc(position.x * scaleX), y: Math.trunc(position.y * scaleY) },
|
||||
})),
|
||||
});
|
||||
const scaledPoses = poses.map((pose) => scalePose(pose, height / inputResolutionHeight, width / inputResolutionWidth));
|
||||
const scaledPoses = poses.map((pose) => scalePose(pose));
|
||||
return scaledPoses;
|
||||
}
|
||||
|
||||
|
|
|
|||
247
src/result.ts
247
src/result.ts
|
|
@ -3,114 +3,151 @@
|
|||
*
|
||||
* Contains all possible detection results
|
||||
*/
|
||||
export interface Result {
|
||||
/** Face results
|
||||
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
|
||||
* Some values may be null if specific model is not enabled
|
||||
*
|
||||
* Array of individual results with one object per detected face
|
||||
* Each result has:
|
||||
* - overal detection confidence value
|
||||
* - box detection confidence value
|
||||
* - mesh detection confidence value
|
||||
* - box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw as array of [x, y, width, height], normalized to range 0..1
|
||||
* - mesh as array of [x, y, z] points of face mesh, normalized to image resolution
|
||||
* - meshRaw as array of [x, y, z] points of face mesh, normalized to range 0..1
|
||||
* - annotations as array of annotated face mesh points
|
||||
* - age as value
|
||||
* - gender as value
|
||||
* - genderConfidence as value
|
||||
* - emotion as array of possible emotions with their individual scores
|
||||
* - iris as distance value
|
||||
* - angle as object with values for roll, yaw and pitch angles
|
||||
* - tensor as Tensor object which contains detected face
|
||||
*/
|
||||
face: Array<{
|
||||
confidence: number,
|
||||
boxConfidence: number,
|
||||
faceConfidence: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
mesh: Array<[number, number, number]>
|
||||
meshRaw: Array<[number, number, number]>
|
||||
annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
|
||||
age: number,
|
||||
gender: string,
|
||||
genderConfidence: number,
|
||||
emotion: Array<{ score: number, emotion: string }>,
|
||||
embedding: Array<number>,
|
||||
iris: number,
|
||||
rotation: {
|
||||
angle: { roll: number, yaw: number, pitch: number },
|
||||
matrix: Array<[number, number, number, number, number, number, number, number, number]>
|
||||
}
|
||||
tensor: any,
|
||||
}>,
|
||||
/** Body results
|
||||
*
|
||||
* Array of individual results with one object per detected body
|
||||
* Each results has:
|
||||
* - body id number
|
||||
* - body part name
|
||||
* - part position with x,y,z coordinates
|
||||
* - body part score value
|
||||
* - body part presence value
|
||||
*/
|
||||
body: Array<{
|
||||
id: number,
|
||||
|
||||
/** Face results
|
||||
* Combined results of face detector, face mesh, age, gender, emotion, embedding, iris models
|
||||
* Some values may be null if specific model is not enabled
|
||||
*
|
||||
* Array of individual results with one object per detected face
|
||||
* Each result has:
|
||||
* - id: face number
|
||||
* - confidence: overal detection confidence value
|
||||
* - boxConfidence: face box detection confidence value
|
||||
* - faceConfidence: face keypoints detection confidence value
|
||||
* - box: face bounding box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw: face bounding box as array of [x, y, width, height], normalized to range 0..1
|
||||
* - mesh: face keypoints as array of [x, y, z] points of face mesh, normalized to image resolution
|
||||
* - meshRaw: face keypoints as array of [x, y, z] points of face mesh, normalized to range 0..1
|
||||
* - annotations: annotated face keypoints as array of annotated face mesh points
|
||||
* - age: age as value
|
||||
* - gender: gender as value
|
||||
* - genderConfidence: gender detection confidence as value
|
||||
* - emotion: emotions as array of possible emotions with their individual scores
|
||||
* - embedding: facial descriptor as array of numerical elements
|
||||
* - iris: iris distance from current viewpoint as distance value
|
||||
* - rotation: face rotiation that contains both angles and matrix used for 3d transformations
|
||||
* - angle: face angle as object with values for roll, yaw and pitch angles
|
||||
* - matrix: 3d transofrmation matrix as array of numeric values
|
||||
* - tensor: face tensor as Tensor object which contains detected face
|
||||
*/
|
||||
export interface Face {
|
||||
id: number
|
||||
confidence: number,
|
||||
boxConfidence: number,
|
||||
faceConfidence: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
mesh: Array<[number, number, number]>
|
||||
meshRaw: Array<[number, number, number]>
|
||||
annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
|
||||
age: number,
|
||||
gender: string,
|
||||
genderConfidence: number,
|
||||
emotion: Array<{ score: number, emotion: string }>,
|
||||
embedding: Array<number>,
|
||||
iris: number,
|
||||
rotation: {
|
||||
angle: { roll: number, yaw: number, pitch: number },
|
||||
matrix: [number, number, number, number, number, number, number, number, number],
|
||||
}
|
||||
tensor: any,
|
||||
}
|
||||
|
||||
/** Body results
|
||||
*
|
||||
* Array of individual results with one object per detected body
|
||||
* Each results has:
|
||||
* - id:body id number
|
||||
* - score: overall detection score
|
||||
* - box: bounding box: x, y, width, height normalized to input image resolution
|
||||
* - boxRaw: bounding box: x, y, width, height normalized to 0..1
|
||||
* - keypoints: array of keypoints
|
||||
* - part: body part name
|
||||
* - position: body part position with x,y,z coordinates
|
||||
* - score: body part score value
|
||||
* - presence: body part presence value
|
||||
*/
|
||||
|
||||
export interface Body {
|
||||
id: number,
|
||||
score: number,
|
||||
box?: [x: number, y: number, width: number, height: number],
|
||||
boxRaw?: [x: number, y: number, width: number, height: number],
|
||||
keypoints: Array<{
|
||||
part: string,
|
||||
position: { x: number, y: number, z: number },
|
||||
score: number,
|
||||
presence: number }>,
|
||||
/** Hand results
|
||||
*
|
||||
* Array of individual results with one object per detected hand
|
||||
* Each result has:
|
||||
* - confidence as value
|
||||
* - box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw as array of [x, y, width, height], normalized to range 0..1
|
||||
* - landmarks as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations as array of annotated face landmark points
|
||||
*/
|
||||
hand: Array<{
|
||||
confidence: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
landmarks: Array<[number, number, number]>,
|
||||
annotations: Array<{ part: string, points: Array<[number, number, number]>[] }>,
|
||||
}>,
|
||||
/** Gesture results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - part: part name and number where gesture was detected: face, iris, body, hand
|
||||
* - gesture: gesture detected
|
||||
*/
|
||||
gesture: Array<
|
||||
{ 'face': number, gesture: string } | { 'iris': number, gesture: string } | { 'body': number, gesture: string } | { 'hand': number, gesture: string }
|
||||
>,
|
||||
/** Object results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - score as value
|
||||
* - label as detected class name
|
||||
* - center as array of [x, y], normalized to image resolution
|
||||
* - centerRaw as array of [x, y], normalized to range 0..1
|
||||
* - box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw as array of [x, y, width, height], normalized to range 0..1
|
||||
*/
|
||||
object: Array<{
|
||||
score: number,
|
||||
strideSize: number,
|
||||
class: number,
|
||||
label: string,
|
||||
center: number[],
|
||||
centerRaw: number[],
|
||||
box: number[],
|
||||
boxRaw: number[],
|
||||
}>,
|
||||
presence: number,
|
||||
}>
|
||||
}
|
||||
|
||||
/** Hand results
|
||||
*
|
||||
* Array of individual results with one object per detected hand
|
||||
* Each result has:
|
||||
* - confidence as value
|
||||
* - box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw as array of [x, y, width, height], normalized to range 0..1
|
||||
* - landmarks as array of [x, y, z] points of hand, normalized to image resolution
|
||||
* - annotations as array of annotated face landmark points
|
||||
*/
|
||||
export interface Hand {
|
||||
id: number,
|
||||
confidence: number,
|
||||
box: [number, number, number, number],
|
||||
boxRaw: [number, number, number, number],
|
||||
landmarks: Array<[number, number, number]>,
|
||||
// annotations: Array<{ part: string, points: Array<[number, number, number]> }>,
|
||||
// annotations: Annotations,
|
||||
annotations: Record<string, Array<{ part: string, points: Array<[number, number, number]> }>>,
|
||||
}
|
||||
|
||||
/** Object results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - score as value
|
||||
* - label as detected class name
|
||||
* - center as array of [x, y], normalized to image resolution
|
||||
* - centerRaw as array of [x, y], normalized to range 0..1
|
||||
* - box as array of [x, y, width, height], normalized to image resolution
|
||||
* - boxRaw as array of [x, y, width, height], normalized to range 0..1
|
||||
*/
|
||||
export interface Item {
|
||||
score: number,
|
||||
strideSize?: number,
|
||||
class: number,
|
||||
label: string,
|
||||
center?: number[],
|
||||
centerRaw?: number[],
|
||||
box: number[],
|
||||
boxRaw: number[],
|
||||
}
|
||||
|
||||
/** Gesture results
|
||||
*
|
||||
* Array of individual results with one object per detected gesture
|
||||
* Each result has:
|
||||
* - part: part name and number where gesture was detected: face, iris, body, hand
|
||||
* - gesture: gesture detected
|
||||
*/
|
||||
export type Gesture =
|
||||
{ 'face': number, gesture: string }
|
||||
| { 'iris': number, gesture: string }
|
||||
| { 'body': number, gesture: string }
|
||||
| { 'hand': number, gesture: string }
|
||||
|
||||
export interface Result {
|
||||
/** {@link Face}: detection & analysis results */
|
||||
face: Array<Face>,
|
||||
/** {@link Body}: detection & analysis results */
|
||||
body: Array<Body>,
|
||||
/** {@link Hand}: detection & analysis results */
|
||||
hand: Array<Hand>,
|
||||
/** {@link Gesture}: detection & analysis results */
|
||||
gesture: Array<Gesture>,
|
||||
/** {@link Object}: detection & analysis results */
|
||||
object: Array<Item>
|
||||
performance: { any },
|
||||
canvas: OffscreenCanvas | HTMLCanvasElement,
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue