diff --git a/.eslintrc.json b/.eslintrc.json index 9005ee77..b8e165df 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -28,8 +28,6 @@ "rules": { "@typescript-eslint/ban-ts-comment": "off", "@typescript-eslint/explicit-module-boundary-types": "off", - "@typescript-eslint/ban-types": "off", - "@typescript-eslint/no-explicit-any": "off", "@typescript-eslint/no-var-requires": "off", "camelcase": "off", "dot-notation": "off", diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a015097..96e7b158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Repository: **** ### **HEAD -> main** 2021/05/22 mandic00@live.com +- rebuild all for release ### **1.9.2** 2021/05/22 mandic00@live.com diff --git a/package.json b/package.json index 9dc1e913..3fb91604 100644 --- a/package.json +++ b/package.json @@ -69,9 +69,9 @@ "chokidar": "^3.5.1", "dayjs": "^1.10.4", "esbuild": "^0.12.1", - "eslint": "^7.26.0", + "eslint": "^7.27.0", "eslint-config-airbnb-base": "^14.2.1", - "eslint-plugin-import": "^2.23.2", + "eslint-plugin-import": "^2.23.3", "eslint-plugin-json": "^3.0.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^5.1.0", diff --git a/src/draw/draw.ts b/src/draw/draw.ts index bbc8adb6..4b7e02f1 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -32,16 +32,16 @@ export interface DrawOptions { lineWidth: number, pointSize: number, roundRect: number, - drawPoints: Boolean, - drawLabels: Boolean, - drawBoxes: Boolean, - drawPolygons: Boolean, - fillPolygons: Boolean, - useDepth: Boolean, - useCurves: Boolean, - bufferedOutput: Boolean, - useRawBoxes: Boolean, - calculateHandBox: Boolean, + drawPoints: boolean, + drawLabels: boolean, + drawBoxes: boolean, + drawPolygons: boolean, + fillPolygons: boolean, + useDepth: boolean, + useCurves: boolean, + bufferedOutput: boolean, + useRawBoxes: boolean, + calculateHandBox: boolean, } export const options: DrawOptions = { @@ -53,16 +53,16 @@ export const options: DrawOptions = { lineWidth: 6, pointSize: 2, roundRect: 28, - drawPoints: false, - drawLabels: true, - drawBoxes: true, - drawPolygons: true, - fillPolygons: false, - useDepth: true, - useCurves: false, - bufferedOutput: false, // not yet implemented - useRawBoxes: false, - calculateHandBox: true, + drawPoints: false, + drawLabels: true, + drawBoxes: true, + drawPolygons: true, + fillPolygons: false, + useDepth: true, + useCurves: false, + bufferedOutput: false, // not yet implemented + useRawBoxes: false, + calculateHandBox: true, }; let bufferedResult: Result; @@ -143,11 +143,11 @@ export async function gesture(inCanvas: HTMLCanvasElement, result: Array 1) && (what[1].length > 0)) { - const person = where[1] > 0 ? `#${where[1]}` : ''; + if ((what.length > 1) && ((what[1] as string).length > 0)) { + const person = where[1] as number > 0 ? `#${where[1]}` : ''; const label = `${where[0]} ${person}: ${what[1]}`; if (localOptions.shadowColor && localOptions.shadowColor !== '') { ctx.fillStyle = localOptions.shadowColor; diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index abd2181c..87b198e0 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -1,8 +1,9 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; import { Body } from '../result'; +import { GraphModel } from '../tfjs/types'; -let model; +let model: GraphModel; type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; @@ -13,10 +14,11 @@ const bodyParts = ['head', 'neck', 'rightShoulder', 'rightElbow', 'rightWrist', export async function load(config) { if (!model) { + // @ts-ignore type mismatch on GraphModel model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath)); - if (!model || !model.modelUrl) log('load model failed:', config.body.modelPath); - else if (config.debug) log('load model:', model.modelUrl); - } else if (config.debug) log('cached model:', model.modelUrl); + if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath); + else if (config.debug) log('load model:', model['modelUrl']); + } else if (config.debug) log('cached model:', model['modelUrl']); return model; } @@ -50,6 +52,7 @@ export async function predict(image, config): Promise { skipped = 0; return new Promise(async (resolve) => { const tensor = tf.tidy(() => { + if (!model.inputs[0].shape) return null; const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false); const enhance = tf.mul(resize, 2); const norm = enhance.sub(1); @@ -75,13 +78,13 @@ export async function predict(image, config): Promise { parts.push({ score: Math.round(100 * score) / 100, part: bodyParts[id], - positionRaw: { - x: x / model.inputs[0].shape[2], // x normalized to 0..1 - y: y / model.inputs[0].shape[1], // y normalized to 0..1 + positionRaw: { // normalized to 0..1 + // @ts-ignore model is not undefined here + x: x / model.inputs[0].shape[2], y: y / model.inputs[0].shape[1], }, - position: { - x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), // x normalized to input image size - y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), // y normalized to input image size + position: { // normalized to input image size + // @ts-ignore model is not undefined here + x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), }, }); } diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index a484bbb7..351dc5e0 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -1,20 +1,22 @@ import { log, join } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import { Tensor, GraphModel } from '../tfjs/types'; -let model; +let model: GraphModel; const last: Array<{ age: number}> = []; let lastCount = 0; let skipped = Number.MAX_SAFE_INTEGER; -type Tensor = typeof tf.Tensor; type DB = Array<{ name: string, source: string, embedding: number[] }>; export async function load(config) { + const modelUrl = join(config.modelBasePath, config.face.description.modelPath); if (!model) { - model = await tf.loadGraphModel(join(config.modelBasePath, config.face.description.modelPath)); - if (!model || !model.modelUrl) log('load model failed:', config.face.description.modelPath); - else if (config.debug) log('load model:', model.modelUrl); - } else if (config.debug) log('cached model:', model.modelUrl); + // @ts-ignore type mismatch for GraphModel + model = await tf.loadGraphModel(modelUrl); + if (!model) log('load model failed:', config.face.description.modelPath); + else if (config.debug) log('load model:', modelUrl); + } else if (config.debug) log('cached model:', modelUrl); return model; } @@ -52,6 +54,7 @@ export function enhance(input): Tensor { // do a tight crop of image and resize it to fit the model const box = [[0.05, 0.15, 0.85, 0.85]]; // empyrical values for top, left, bottom, right // const box = [[0.0, 0.0, 1.0, 1.0]]; // basically no crop for test + if (!model.inputs[0].shape) return null; // model has no shape so no point continuing const crop = (tensor.shape.length === 3) ? tf.image.cropAndResize(tf.expandDims(tensor, 0), box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]) // add batch dimension if missing : tf.image.cropAndResize(tensor, box, [0], [model.inputs[0].shape[2], model.inputs[0].shape[1]]); diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index 82ba2e86..94f534cb 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -1,9 +1,10 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as anchors from './anchors'; +import { Tensor, GraphModel } from '../tfjs/types'; export class HandDetector { - model: any; // tf.GraphModel + model: GraphModel; anchors: number[][]; anchorsTensor: typeof tf.Tensor; inputSize: number; @@ -14,6 +15,7 @@ export class HandDetector { this.model = model; this.anchors = anchors.anchors.map((anchor) => [anchor.x, anchor.y]); this.anchorsTensor = tf.tensor2d(this.anchors); + // @ts-ignore model is not undefined here this.inputSize = this.model?.inputs[0].shape[2]; this.inputSizeTensor = tf.tensor1d([this.inputSize, this.inputSize]); this.doubleInputSizeTensor = tf.tensor1d([this.inputSize * 2, this.inputSize * 2]); @@ -39,7 +41,7 @@ export class HandDetector { } async getBoxes(input, config) { - const batched = this.model.predict(input); + const batched = this.model.predict(input) as Tensor; const predictions = batched.squeeze(); batched.dispose(); const scoresT = tf.tidy(() => tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1])).squeeze()); @@ -52,7 +54,7 @@ export class HandDetector { scoresT.dispose(); filteredT.dispose(); - const hands: Array<{ box: any, palmLandmarks: any, confidence: number }> = []; // box and lardmarks are tensors here + const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = []; for (const index of filtered) { if (scores[index] >= config.hand.minConfidence) { const matchingBox = tf.slice(boxes, [index, 0], [1, -1]); diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index 75b3af8c..269deaf6 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -2,6 +2,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as box from './box'; import * as util from './util'; import * as detector from './handdetector'; +import { Tensor, GraphModel } from '../tfjs/types'; const palmBoxEnlargeFactor = 5; // default 3 const handBoxEnlargeFactor = 1.65; // default 1.65 @@ -11,7 +12,7 @@ const palmLandmarksMiddleFingerBase = 2; export class HandPipeline { handDetector: detector.HandDetector; - handPoseModel: any; // tf.GraphModel + handPoseModel: GraphModel; inputSize: number; storedBoxes: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number } | null>; skipped: number; @@ -20,6 +21,7 @@ export class HandPipeline { constructor(handDetector, handPoseModel) { this.handDetector = handDetector; this.handPoseModel = handPoseModel; + // @ts-ignore model is not undefined here this.inputSize = this.handPoseModel?.inputs[0].shape[2]; this.storedBoxes = []; this.skipped = 0; @@ -96,7 +98,7 @@ export class HandPipeline { // for (const possible of boxes) this.storedBoxes.push(possible); if (this.storedBoxes.length > 0) useFreshBox = true; } - const hands: Array<{}> = []; + const hands: Array<{ landmarks?: number[], confidence: number, box: { topLeft: number[], bottomRight: number[] } }> = []; // go through working set of boxes for (let i = 0; i < this.storedBoxes.length; i++) { @@ -113,7 +115,7 @@ export class HandPipeline { const handImage = croppedInput.div(255); croppedInput.dispose(); rotatedImage.dispose(); - const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage); + const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array; handImage.dispose(); const confidence = confidenceT.dataSync()[0]; confidenceT.dispose(); diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 9499cfbf..96f27eaa 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -3,6 +3,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as handdetector from './handdetector'; import * as handpipeline from './handpipeline'; import { Hand } from '../result'; +import { GraphModel } from '../tfjs/types'; const meshAnnotations = { thumb: [1, 2, 3, 4], @@ -13,9 +14,9 @@ const meshAnnotations = { palmBase: [0], }; -let handDetectorModel; -let handPoseModel; -let handPipeline; +let handDetectorModel: GraphModel | null; +let handPoseModel: GraphModel | null; +let handPipeline: handpipeline.HandPipeline; export async function predict(input, config): Promise { const predictions = await handPipeline.estimateHands(input, config); @@ -25,6 +26,7 @@ export async function predict(input, config): Promise { const annotations = {}; if (predictions[i].landmarks) { for (const key of Object.keys(meshAnnotations)) { + // @ts-ignore landmarks are not undefined annotations[key] = meshAnnotations[key].map((index) => predictions[i].landmarks[index]); } } @@ -40,26 +42,28 @@ export async function predict(input, config): Promise { (predictions[i].box.bottomRight[0] - predictions[i].box.topLeft[0]) / input.shape[2], (predictions[i].box.bottomRight[1] - predictions[i].box.topLeft[1]) / input.shape[1], ]; - hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, landmarks: predictions[i].landmarks, annotations }); + const landmarks = predictions[i].landmarks as number[]; + hands.push({ id: i, confidence: Math.round(100 * predictions[i].confidence) / 100, box, boxRaw, landmarks, annotations }); } return hands; } -export async function load(config): Promise<[Object, Object]> { +export async function load(config): Promise<[unknown, unknown]> { if (!handDetectorModel || !handPoseModel) { + // @ts-ignore type mismatch on GraphModel [handDetectorModel, handPoseModel] = await Promise.all([ config.hand.enabled ? tf.loadGraphModel(join(config.modelBasePath, config.hand.detector.modelPath), { fromTFHub: config.hand.detector.modelPath.includes('tfhub.dev') }) : null, config.hand.landmarks ? tf.loadGraphModel(join(config.modelBasePath, config.hand.skeleton.modelPath), { fromTFHub: config.hand.skeleton.modelPath.includes('tfhub.dev') }) : null, ]); if (config.hand.enabled) { - if (!handDetectorModel || !handDetectorModel.modelUrl) log('load model failed:', config.hand.detector.modelPath); - else if (config.debug) log('load model:', handDetectorModel.modelUrl); - if (!handPoseModel || !handPoseModel.modelUrl) log('load model failed:', config.hand.skeleton.modelPath); - else if (config.debug) log('load model:', handPoseModel.modelUrl); + if (!handDetectorModel || !handDetectorModel['modelUrl']) log('load model failed:', config.hand.detector.modelPath); + else if (config.debug) log('load model:', handDetectorModel['modelUrl']); + if (!handPoseModel || !handPoseModel['modelUrl']) log('load model failed:', config.hand.skeleton.modelPath); + else if (config.debug) log('load model:', handPoseModel['modelUrl']); } } else { - if (config.debug) log('cached model:', handDetectorModel.modelUrl); - if (config.debug) log('cached model:', handPoseModel.modelUrl); + if (config.debug) log('cached model:', handDetectorModel['modelUrl']); + if (config.debug) log('cached model:', handPoseModel['modelUrl']); } const handDetector = new handdetector.HandDetector(handDetectorModel); handPipeline = new handpipeline.HandPipeline(handDetector, handPoseModel); diff --git a/src/human.ts b/src/human.ts index 18921c6e..d67de042 100644 --- a/src/human.ts +++ b/src/human.ts @@ -18,9 +18,7 @@ import * as image from './image/image'; import * as draw from './draw/draw'; import * as sample from './sample'; import * as app from '../package.json'; - -/** Generic Tensor object type */ -export type Tensor = typeof tf.Tensor; +import { Tensor } from './tfjs/types'; export type { Config } from './config'; export type { Result, Face, Hand, Body, Item, Gesture } from './result'; @@ -36,7 +34,7 @@ export type Error = { error: string }; export type TensorFlow = typeof tf; /** Generic Model object type, holds instance of individual models */ -type Model = Object; +type Model = unknown; /** * **Human** library main class @@ -114,7 +112,7 @@ export class Human { /** Platform and agent information detected by Human */ sysinfo: { platform: string, agent: string }; /** Performance object that contains values for all recently performed operations */ - perf: any; // perf members are dynamically defined as needed + perf: Record; // perf members are dynamically defined as needed #numTensors: number; #analyzeMemoryLeaks: boolean; #checkSanity: boolean; @@ -128,7 +126,7 @@ export class Human { * Creates instance of Human library that is futher used for all operations * - @param userConfig: {@link Config} */ - constructor(userConfig: Config | Object = {}) { + constructor(userConfig: Config | Record = {}) { this.tf = tf; this.draw = draw; this.version = app.version; @@ -215,6 +213,7 @@ export class Human { */ // eslint-disable-next-line class-methods-use-this enhance(input: Tensor): Tensor | null { + // @ts-ignore type mismach for Tensor return faceres.enhance(input); } @@ -233,7 +232,7 @@ export class Human { /** Load method preloads all configured models on-demand * - Not explicitly required as any required model is load implicitly on it's first run */ - async load(userConfig: Config | Object = {}) { + async load(userConfig: Config | Record = {}) { this.state = 'load'; const timeStamp = now(); if (userConfig) this.config = mergeDeep(this.config, userConfig); @@ -287,7 +286,7 @@ export class Human { } const current = Math.trunc(now() - timeStamp); - if (current > (this.perf.load || 0)) this.perf.load = current; + if (current > (this.perf.load as number || 0)) this.perf.load = current; } // check if backend needs initialization if it changed @@ -385,7 +384,7 @@ export class Human { * - Run inference for all configured models * - Process and return result: {@link Result} */ - async detect(input: Input, userConfig: Config | Object = {}): Promise { + async detect(input: Input, userConfig: Config | Record = {}): Promise { // detection happens inside a promise return new Promise(async (resolve) => { this.state = 'config'; @@ -442,7 +441,7 @@ export class Human { this.config.skipFrame = await this.#skipFrame(process.tensor); if (!this.perf.frames) this.perf.frames = 0; if (!this.perf.cached) this.perf.cached = 0; - this.perf.frames++; + (this.perf.frames as number)++; // @ts-ignore hidden dynamic property that is not part of definitions if (this.config.skipFrame) this.perf.cached++; this.perf.changed = Math.trunc(now() - timeStamp); @@ -629,7 +628,7 @@ export class Human { * - can take significant time on startup * - only used for `webgl` and `humangl` backends */ - async warmup(userConfig: Config | Object = {}): Promise { + async warmup(userConfig: Config | Record = {}): Promise { const t0 = now(); if (userConfig) this.config = mergeDeep(this.config, userConfig); if (!this.config.warmup || this.config.warmup === 'none') return { error: 'null' }; diff --git a/src/posenet/poses.ts b/src/posenet/poses.ts index 296f2926..9a082f28 100644 --- a/src/posenet/poses.ts +++ b/src/posenet/poses.ts @@ -127,7 +127,9 @@ export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxD // The top element in the queue is the next root candidate. const root = queue.dequeue(); // Part-based non-maximum suppression: We reject a root candidate if it is within a disk of `nmsRadius` pixels from the corresponding part of a previously detected instance. + // @ts-ignore this one is tree walk const rootImageCoords = utils.getImageCoords(root.part, outputStride, offsets); + // @ts-ignore this one is tree walk if (withinRadius(poses, rootImageCoords, root.part.id)) continue; // Else start a new detection instance at the position of the root. let keypoints = decodePose(root, scores, offsets, displacementsFwd, displacementsBwd); diff --git a/src/posenet/utils.ts b/src/posenet/utils.ts index 7c7bb221..aea0092a 100644 --- a/src/posenet/utils.ts +++ b/src/posenet/utils.ts @@ -49,9 +49,9 @@ export function scalePoses(poses, [height, width], [inputResolutionHeight, input // algorithm based on Coursera Lecture from Algorithms, Part 1: https://www.coursera.org/learn/algorithms-part1/lecture/ZjoSM/heapsort export class MaxHeap { - priorityQueue: Array; // don't touch + priorityQueue: Array; // don't touch numberOfElements: number; - getElementValue: any; // function call + getElementValue: unknown; // function call constructor(maxSize, getElementValue) { this.priorityQueue = new Array(maxSize); @@ -98,6 +98,7 @@ export class MaxHeap { } getValueAt(i) { + // @ts-ignore getter is of unknown type return this.getElementValue(this.priorityQueue[i]); } diff --git a/src/profile.ts b/src/profile.ts index f7e034c2..64c12f65 100644 --- a/src/profile.ts +++ b/src/profile.ts @@ -2,16 +2,19 @@ import { log } from './helpers'; export const data = {}; -export function run(modelName: string, profileData: any): void { // profileData is tfjs internal type +export function run(modelName: string, profileData: Record): void { // profileData is tfjs internal type if (!profileData || !profileData.kernels) return; const maxDetected = 5; + // @ts-ignore profileData.kernels is tfjs internal type const time = profileData.kernels .filter((a) => a.kernelTimeMs > 0) .reduce((a, b) => a += b.kernelTimeMs, 0); + // @ts-ignore profileData.kernels is tfjs internal type const slowest = profileData.kernels .map((a, i) => { a.id = i; return a; }) .filter((a) => a.kernelTimeMs > 0) .sort((a, b) => b.kernelTimeMs - a.kernelTimeMs); + // @ts-ignore profileData.kernels is tfjs internal type const largest = profileData.kernels .map((a, i) => { a.id = i; return a; }) .filter((a) => a.totalBytesSnapshot > 0) @@ -23,7 +26,7 @@ export function run(modelName: string, profileData: any): void { // profileData newBytes: profileData.newBytes, newTensors: profileData.newTensors, peakBytes: profileData.peakBytes, - numKernelOps: profileData.kernels.length, + numKernelOps: (profileData['kernels'] as Array).length, timeKernelOps: time, slowestKernelOps: slowest, largestKernelOps: largest, diff --git a/src/result.ts b/src/result.ts index da1d0f81..8e54fd71 100644 --- a/src/result.ts +++ b/src/result.ts @@ -99,9 +99,7 @@ export interface Hand { confidence: number, box: [number, number, number, number], boxRaw: [number, number, number, number], - landmarks: Array<[number, number, number]>, - // annotations: Array<{ part: string, points: Array<[number, number, number]> }>, - // annotations: Annotations, + landmarks: number[], annotations: Record }>>, } @@ -151,7 +149,7 @@ export interface Result { gesture: Array, /** {@link Object}: detection & analysis results */ object: Array - performance: { any }, + performance: Record, canvas: OffscreenCanvas | HTMLCanvasElement, timestamp: number, } diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index 596b9278..5c3db809 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -5,7 +5,7 @@ export const config = { name: 'humangl', priority: 99, canvas: null, - gl: null, + gl: null, width: 1024, height: 1024, webGLattr: { // https://www.khronos.org/registry/webgl/specs/latest/1.0/#5.2 diff --git a/src/tfjs/types.ts b/src/tfjs/types.ts new file mode 100644 index 00000000..3189b32f --- /dev/null +++ b/src/tfjs/types.ts @@ -0,0 +1,3 @@ +// export common types +export { Tensor } from '@tensorflow/tfjs-core/dist/index'; +export { GraphModel } from '@tensorflow/tfjs-converter/dist/index';