mirror of https://github.com/vladmandic/human
enhance strong typing
Vladimir Mandic
parent
a21f9b2a06
commit
b192445071
|
|
@ -9,14 +9,14 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/05/22 mandic00@live.com
|
||||
|
||||
|
||||
### **1.9.2** 2021/05/22 mandic00@live.com
|
||||
|
||||
- add id and boxraw on missing objects
|
||||
- restructure results strong typing
|
||||
|
||||
### **origin/main** 2021/05/21 mandic00@live.com
|
||||
|
||||
|
||||
### **1.9.1** 2021/05/21 mandic00@live.com
|
||||
|
||||
- caching improvements
|
||||
|
|
|
|||
|
|
@ -96,13 +96,14 @@ function rect(ctx, x, y, width, height, localOptions) {
|
|||
ctx.stroke();
|
||||
}
|
||||
|
||||
function lines(ctx, points: [number, number, number][] = [], localOptions) {
|
||||
function lines(ctx, points: [number, number, number?][] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(points[0][0], points[0][1]);
|
||||
for (const pt of points) {
|
||||
ctx.strokeStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && pt[2] ? `rgba(${127.5 + (2 * pt[2])}, ${127.5 - (2 * pt[2])}, 255, 0.3)` : localOptions.color;
|
||||
const z = pt[2] || 0;
|
||||
ctx.strokeStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
|
||||
ctx.lineTo(pt[0], Math.round(pt[1]));
|
||||
}
|
||||
ctx.stroke();
|
||||
|
|
@ -112,7 +113,7 @@ function lines(ctx, points: [number, number, number][] = [], localOptions) {
|
|||
}
|
||||
}
|
||||
|
||||
function curves(ctx, points: [number, number, number][] = [], localOptions) {
|
||||
function curves(ctx, points: [number, number, number?][] = [], localOptions) {
|
||||
if (points === undefined || points.length === 0) return;
|
||||
if (!localOptions.useCurves || points.length <= 2) {
|
||||
lines(ctx, points, localOptions);
|
||||
|
|
@ -142,8 +143,8 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array<Gesture
|
|||
ctx.fillStyle = localOptions.color;
|
||||
let i = 1;
|
||||
for (let j = 0; j < result.length; j++) {
|
||||
let where:any[] = [];
|
||||
let what:any[] = [];
|
||||
let where: any[] = []; // what&where is a record
|
||||
let what: any[] = []; // what&where is a record
|
||||
[where, what] = Object.entries(result[j]);
|
||||
if ((what.length > 1) && (what[1].length > 0)) {
|
||||
const person = where[1] > 0 ? `#${where[1]}` : '';
|
||||
|
|
@ -271,7 +272,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
|
|||
}
|
||||
if (localOptions.drawPoints) {
|
||||
for (let pt = 0; pt < result[i].keypoints.length; pt++) {
|
||||
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * result[i].keypoints[pt].position.z)}, ${127.5 - (2 * result[i].keypoints[pt].position.z)}, 255, 0.5)` : localOptions.color;
|
||||
ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position.z ? `rgba(${127.5 + (2 * (result[i].keypoints[pt].position.z || 0))}, ${127.5 - (2 * (result[i].keypoints[pt].position.z || 0))}, 255, 0.5)` : localOptions.color;
|
||||
point(ctx, result[i].keypoints[pt].position.x, result[i].keypoints[pt].position.y, 0, localOptions);
|
||||
}
|
||||
}
|
||||
|
|
@ -286,7 +287,7 @@ export async function body(inCanvas: HTMLCanvasElement, result: Array<Body>, dra
|
|||
}
|
||||
if (localOptions.drawPolygons && result[i].keypoints) {
|
||||
let part;
|
||||
const points: any[] = [];
|
||||
const points: [number, number, number?][] = [];
|
||||
// shoulder line
|
||||
points.length = 0;
|
||||
part = result[i].keypoints.find((a) => a.part === 'leftShoulder');
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@ import * as tf from '../../dist/tfjs.esm.js';
|
|||
import { Body } from '../result';
|
||||
|
||||
let model;
|
||||
let keypoints: Array<any> = [];
|
||||
|
||||
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } };
|
||||
|
||||
let keypoints: Array<Keypoints> = [];
|
||||
let skipped = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', 'chest', 'leftShoulder', 'leftElbow', 'leftWrist', 'pelvis', 'rightHip', 'rightKnee', 'rightAnkle', 'leftHip', 'leftKnee', 'leftAnkle'];
|
||||
|
|
@ -41,7 +44,8 @@ function max2d(inputs, minScore) {
|
|||
export async function predict(image, config): Promise<Body[]> {
|
||||
if ((skipped < config.body.skipFrames) && config.skipFrame && Object.keys(keypoints).length > 0) {
|
||||
skipped++;
|
||||
return keypoints;
|
||||
const score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
|
||||
return [{ id: 0, score, keypoints }];
|
||||
}
|
||||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
|
|
@ -57,7 +61,7 @@ export async function predict(image, config): Promise<Body[]> {
|
|||
tensor.dispose();
|
||||
|
||||
if (resT) {
|
||||
const parts: Array<{ id, score, part, position: { x, y }, positionRaw: { xRaw, yRaw} }> = [];
|
||||
const parts: Array<Keypoints> = [];
|
||||
const squeeze = resT.squeeze();
|
||||
tf.dispose(resT);
|
||||
// body parts are basically just a stack of 2d tensors
|
||||
|
|
@ -69,12 +73,11 @@ export async function predict(image, config): Promise<Body[]> {
|
|||
const [x, y, score] = max2d(stack[id], config.body.minConfidence);
|
||||
if (score > config.body.minConfidence) {
|
||||
parts.push({
|
||||
id,
|
||||
score: Math.round(100 * score) / 100,
|
||||
part: bodyParts[id],
|
||||
positionRaw: {
|
||||
xRaw: x / model.inputs[0].shape[2], // x normalized to 0..1
|
||||
yRaw: y / model.inputs[0].shape[1], // y normalized to 0..1
|
||||
x: x / model.inputs[0].shape[2], // x normalized to 0..1
|
||||
y: y / model.inputs[0].shape[1], // y normalized to 0..1
|
||||
},
|
||||
position: {
|
||||
x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ const calculateFaceAngle = (face, image_size): { angle: { pitch: number, yaw: nu
|
|||
return { angle, matrix };
|
||||
};
|
||||
|
||||
export const detectFace = async (parent, input): Promise<any> => {
|
||||
export const detectFace = async (parent, input): Promise<Face[]> => {
|
||||
// run facemesh, includes blazeface and iris
|
||||
// eslint-disable-next-line no-async-promise-executor
|
||||
let timeStamp;
|
||||
|
|
|
|||
|
|
@ -3,12 +3,12 @@ import * as box from './box';
|
|||
import * as anchors from './anchors';
|
||||
|
||||
export class HandDetector {
|
||||
model: any;
|
||||
anchors: any;
|
||||
anchorsTensor: any;
|
||||
model: any; // tf.GraphModel
|
||||
anchors: number[][];
|
||||
anchorsTensor: typeof tf.Tensor;
|
||||
inputSize: number;
|
||||
inputSizeTensor: any;
|
||||
doubleInputSizeTensor: any;
|
||||
inputSizeTensor: typeof tf.Tensor;
|
||||
doubleInputSizeTensor: typeof tf.Tensor;
|
||||
|
||||
constructor(model) {
|
||||
this.model = model;
|
||||
|
|
@ -52,7 +52,7 @@ export class HandDetector {
|
|||
|
||||
scoresT.dispose();
|
||||
filteredT.dispose();
|
||||
const hands: Array<{ box: any, palmLandmarks: any, confidence: number }> = [];
|
||||
const hands: Array<{ box: any, palmLandmarks: any, confidence: number }> = []; // box and lardmarks are tensors here
|
||||
for (const index of filtered) {
|
||||
if (scores[index] >= config.hand.minConfidence) {
|
||||
const matchingBox = tf.slice(boxes, [index, 0], [1, -1]);
|
||||
|
|
@ -67,13 +67,13 @@ export class HandDetector {
|
|||
return hands;
|
||||
}
|
||||
|
||||
async estimateHandBounds(input, config) {
|
||||
async estimateHandBounds(input, config): Promise<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }[]> {
|
||||
const inputHeight = input.shape[1];
|
||||
const inputWidth = input.shape[2];
|
||||
const image = tf.tidy(() => input.resizeBilinear([this.inputSize, this.inputSize]).div(127.5).sub(1));
|
||||
const predictions = await this.getBoxes(image, config);
|
||||
image.dispose();
|
||||
const hands: Array<{}> = [];
|
||||
const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
|
||||
if (!predictions || predictions.length === 0) return hands;
|
||||
for (const prediction of predictions) {
|
||||
const boxes = prediction.box.dataSync();
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as box from './box';
|
||||
import * as util from './util';
|
||||
import * as detector from './handdetector';
|
||||
|
||||
const palmBoxEnlargeFactor = 5; // default 3
|
||||
const handBoxEnlargeFactor = 1.65; // default 1.65
|
||||
|
|
@ -9,17 +10,17 @@ const palmLandmarksPalmBase = 0;
|
|||
const palmLandmarksMiddleFingerBase = 2;
|
||||
|
||||
export class HandPipeline {
|
||||
handDetector: any;
|
||||
landmarkDetector: any;
|
||||
handDetector: detector.HandDetector;
|
||||
handPoseModel: any; // tf.GraphModel
|
||||
inputSize: number;
|
||||
storedBoxes: any;
|
||||
storedBoxes: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number } | null>;
|
||||
skipped: number;
|
||||
detectedHands: number;
|
||||
|
||||
constructor(handDetector, landmarkDetector) {
|
||||
constructor(handDetector, handPoseModel) {
|
||||
this.handDetector = handDetector;
|
||||
this.landmarkDetector = landmarkDetector;
|
||||
this.inputSize = this.landmarkDetector?.inputs[0].shape[2];
|
||||
this.handPoseModel = handPoseModel;
|
||||
this.inputSize = this.handPoseModel?.inputs[0].shape[2];
|
||||
this.storedBoxes = [];
|
||||
this.skipped = 0;
|
||||
this.detectedHands = 0;
|
||||
|
|
@ -112,7 +113,7 @@ export class HandPipeline {
|
|||
const handImage = croppedInput.div(255);
|
||||
croppedInput.dispose();
|
||||
rotatedImage.dispose();
|
||||
const [confidenceT, keypoints] = await this.landmarkDetector.predict(handImage);
|
||||
const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage);
|
||||
handImage.dispose();
|
||||
const confidence = confidenceT.dataSync()[0];
|
||||
confidenceT.dispose();
|
||||
|
|
@ -123,7 +124,7 @@ export class HandPipeline {
|
|||
keypointsReshaped.dispose();
|
||||
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
|
||||
const nextBoundingBox = this.getBoxForHandLandmarks(coords);
|
||||
this.storedBoxes[i] = nextBoundingBox;
|
||||
this.storedBoxes[i] = { ...nextBoundingBox, confidence };
|
||||
const result = {
|
||||
landmarks: coords,
|
||||
confidence,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { log, now, mergeDeep } from './helpers';
|
||||
import { Config, defaults } from './config';
|
||||
import { Result } from './result';
|
||||
import { Result, Gesture } from './result';
|
||||
import * as sysinfo from './sysinfo';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as backend from './tfjs/backend';
|
||||
|
|
@ -114,7 +114,7 @@ export class Human {
|
|||
/** Platform and agent information detected by Human */
|
||||
sysinfo: { platform: string, agent: string };
|
||||
/** Performance object that contains values for all recently performed operations */
|
||||
perf: any;
|
||||
perf: any; // perf members are dynamically defined as needed
|
||||
#numTensors: number;
|
||||
#analyzeMemoryLeaks: boolean;
|
||||
#checkSanity: boolean;
|
||||
|
|
@ -449,9 +449,10 @@ export class Human {
|
|||
this.analyze('Check Changed:');
|
||||
|
||||
// prepare where to store model results
|
||||
// keep them with weak typing as it can be promise or not
|
||||
let faceRes;
|
||||
let bodyRes;
|
||||
let handRes;
|
||||
let faceRes;
|
||||
let objectRes;
|
||||
let current;
|
||||
|
||||
|
|
@ -520,7 +521,7 @@ export class Human {
|
|||
tf.dispose(process.tensor);
|
||||
|
||||
// run gesture analysis last
|
||||
let gestureRes: any[] = [];
|
||||
let gestureRes: Gesture[] = [];
|
||||
if (this.config.gesture.enabled) {
|
||||
timeStamp = now();
|
||||
gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)];
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ async function process(res, inputSize, outputShape, config) {
|
|||
// unnecessary boxes and run nms only on good candidates (basically it just does IOU analysis as scores are already filtered)
|
||||
const nmsBoxes = results.map((a) => [a.boxRaw[1], a.boxRaw[0], a.boxRaw[3], a.boxRaw[2]]); // switches coordinates from x,y to y,x as expected by tf.nms
|
||||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx: any[] = [];
|
||||
let nmsIdx: Array<number> = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
|
||||
nmsIdx = nms.dataSync();
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ function getInstanceScore(existingPoses, keypoints) {
|
|||
}
|
||||
|
||||
export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxDetected, minConfidence) {
|
||||
const poses: Array<{ keypoints: any, box: any, score: number }> = [];
|
||||
const poses: Array<{ keypoints, box: [number, number, number, number], score: number }> = [];
|
||||
const queue = buildPartWithScoreQueue(minConfidence, scores);
|
||||
// Generate at most maxDetected object instances per image in decreasing root part score order.
|
||||
while (poses.length < maxDetected && !queue.empty()) {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export function getAdjacentKeyPoints(keypoints, minConfidence) {
|
|||
}, []);
|
||||
}
|
||||
|
||||
export function getBoundingBox(keypoints) {
|
||||
export function getBoundingBox(keypoints): [number, number, number, number] {
|
||||
const coord = keypoints.reduce(({ maxX, maxY, minX, minY }, { position: { x, y } }) => ({
|
||||
maxX: Math.max(maxX, x),
|
||||
maxY: Math.max(maxY, y),
|
||||
|
|
@ -49,9 +49,9 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input
|
|||
|
||||
// algorithm based on Coursera Lecture from Algorithms, Part 1: https://www.coursera.org/learn/algorithms-part1/lecture/ZjoSM/heapsort
|
||||
export class MaxHeap {
|
||||
priorityQueue: any;
|
||||
priorityQueue: Array<any>; // don't touch
|
||||
numberOfElements: number;
|
||||
getElementValue: any;
|
||||
getElementValue: any; // function call
|
||||
|
||||
constructor(maxSize, getElementValue) {
|
||||
this.priorityQueue = new Array(maxSize);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { log } from './helpers';
|
|||
|
||||
export const data = {};
|
||||
|
||||
export function run(modelName: string, profileData: any): void {
|
||||
export function run(modelName: string, profileData: any): void { // profileData is tfjs internal type
|
||||
if (!profileData || !profileData.kernels) return;
|
||||
const maxDetected = 5;
|
||||
const time = profileData.kernels
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@
|
|||
* - matrix: 3d transofrmation matrix as array of numeric values
|
||||
* - tensor: face tensor as Tensor object which contains detected face
|
||||
*/
|
||||
import { Tensor } from '../dist/tfjs.esm.js';
|
||||
|
||||
export interface Face {
|
||||
id: number
|
||||
confidence: number,
|
||||
|
|
@ -50,7 +52,7 @@ export interface Face {
|
|||
angle: { roll: number, yaw: number, pitch: number },
|
||||
matrix: [number, number, number, number, number, number, number, number, number],
|
||||
}
|
||||
tensor: any,
|
||||
tensor: typeof Tensor,
|
||||
}
|
||||
|
||||
/** Body results
|
||||
|
|
@ -75,7 +77,8 @@ export interface Body {
|
|||
boxRaw?: [x: number, y: number, width: number, height: number],
|
||||
keypoints: Array<{
|
||||
part: string,
|
||||
position: { x: number, y: number, z: number },
|
||||
position: { x: number, y: number, z?: number },
|
||||
positionRaw?: { x: number, y: number, z?: number },
|
||||
score: number,
|
||||
presence?: number,
|
||||
}>
|
||||
|
|
|
|||
Loading…
Reference in New Issue