diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f44bc6f..bd8213ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human -Version: **1.4.0** +Version: **1.4.1** Description: **Human: AI-powered 3D Face Detection, Face Description & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,8 +9,9 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/04/08 mandic00@live.com +### **1.4.1** 2021/04/09 mandic00@live.com +- add modelbasepath option ### **1.3.5** 2021/04/06 mandic00@live.com diff --git a/README.md b/README.md index 21b18724..8fae44ce 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,10 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) fo ## Demos -- [**Demo Application**](https://vladmandic.github.io/human/demo/index.html) +- [**Main Application**](https://vladmandic.github.io/human/demo/index.html) - [**Face Extraction, Description, Identification and Matching**](https://vladmandic.github.io/human/demo/facematch.html) - [**Face Extraction and 3D Rendering**](https://vladmandic.github.io/human/demo/face3d.html) +- [**Details on Demo Applications**](https://github.com/vladmandic/human/wiki/Demos) ## Project pages @@ -47,7 +48,6 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) fo ## Wiki pages - [**Home**](https://github.com/vladmandic/human/wiki) -- [**Demos**](https://github.com/vladmandic/human/wiki/Demos) - [**Installation**](https://github.com/vladmandic/human/wiki/Install) - [**Usage & Functions**](https://github.com/vladmandic/human/wiki/Usage) - [**Configuration Details**](https://github.com/vladmandic/human/wiki/Configuration) @@ -185,7 +185,7 @@ For more info, see [**Configuration Details**](https://github.com/vladmandic/hum

-`Human` library is written in `TypeScript` [4.3](https://www.typescriptlang.org/docs/handbook/intro.html) +`Human` library is written in `TypeScript` [4.2](https://www.typescriptlang.org/docs/handbook/intro.html) Conforming to `JavaScript` [ECMAScript version 2020](https://www.ecma-international.org/ecma-262/11.0/index.html) standard Build target is `JavaScript` **EMCAScript version 2018** diff --git a/demo/helpers/menu.js b/demo/helpers/menu.js index 999a1053..68e33dc0 100644 --- a/demo/helpers/menu.js +++ b/demo/helpers/menu.js @@ -1,5 +1,3 @@ -// @ts-nocheck - let instance = 0; let CSScreated = false; diff --git a/demo/index.js b/demo/index.js index b3e4f76f..51c9e7d6 100644 --- a/demo/index.js +++ b/demo/index.js @@ -5,6 +5,7 @@ import Menu from './helpers/menu.js'; import GLBench from './helpers/gl-bench.js'; const userConfig = { backend: 'webgl' }; // add any user configuration overrides +let human; /* const userConfig = { @@ -42,6 +43,7 @@ const ui = { console: true, // log messages to browser console maxFPSframes: 10, // keep fps history for how many frames modelsPreload: true, // preload human models on startup + modelsWarmup: true, // warmup human models on startup busy: false, // internal camera busy flag menuWidth: 0, // internal menuHeight: 0, // internal @@ -89,12 +91,6 @@ function status(msg) { if (div) div.innerText = msg; } -const human = new Human(userConfig); -if (typeof tf !== 'undefined') { - log('TensorFlow external version:', tf.version); - human.tf = tf; // use externally loaded version of tfjs -} - const compare = { enabled: false, original: null }; async function calcSimmilariry(result) { document.getElementById('compare-container').style.display = compare.enabled ? 'block' : 'none'; @@ -564,20 +560,57 @@ async function drawWarmup(res) { async function main() { log('demo starting ...'); + + // parse url search params + const params = new URLSearchParams(location.search); + log('url options:', params.toString()); + if (params.has('worker')) { + ui.useWorker = JSON.parse(params.get('worker')); + log('overriding worker:', ui.useWorker); + } + if (params.has('backend')) { + userConfig.backend = JSON.parse(params.get('backend')); + log('overriding backend:', userConfig.backend); + } + if (params.has('preload')) { + ui.modelsPreload = JSON.parse(params.get('preload')); + log('overriding preload:', ui.modelsPreload); + } + if (params.has('warmup')) { + ui.modelsWarmup = JSON.parse(params.get('warmup')); + log('overriding warmup:', ui.modelsWarmup); + } + + // create instance of human + human = new Human(userConfig); + if (typeof tf !== 'undefined') { + log('TensorFlow external version:', tf.version); + human.tf = tf; // use externally loaded version of tfjs + } + + // setup main menu setupMenu(); document.getElementById('log').innerText = `Human: version ${human.version}`; + + // preload models if (ui.modelsPreload && !ui.useWorker) { status('loading'); await human.load(userConfig); // this is not required, just pre-loads all models const loaded = Object.keys(human.models).filter((a) => human.models[a]); log('demo loaded models:', loaded); } - if (!ui.useWorker) { + + // warmup models + if (ui.modelsWarmup && !ui.useWorker) { status('initializing'); const res = await human.warmup(userConfig); // this is not required, just pre-warms all models for faster initial inference if (res && res.canvas && ui.drawWarmup) await drawWarmup(res); } + + // setup camera await setupCamera(); + + // ready status('human: ready'); document.getElementById('loader').style.display = 'none'; document.getElementById('play').style.display = 'block'; diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index b174e920..c1b62c72 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -64,8 +64,7 @@ export const hand = (res) => { for (let i = 0; i < res.length; i++) { const fingers: Array<{ name: string, position: number }> = []; for (const [finger, pos] of Object.entries(res[i]['annotations'])) { - // @ts-ignore - if (finger !== 'palmBase') fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger + if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger } if (fingers && fingers.length > 0) { const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); diff --git a/src/human.ts b/src/human.ts index c2707d2c..dda6eae4 100644 --- a/src/human.ts +++ b/src/human.ts @@ -124,7 +124,7 @@ export class Human { faceres: null, }; // export access to image processing - // @ts-ignore + // @ts-ignore // typescript cannot infer type this.image = (input: Input) => image.process(input, this.config); // export raw access to underlying models this.classes = { @@ -214,9 +214,9 @@ export class Human { this.models.gender, this.models.emotion, this.models.embedding, - // @ts-ignore + // @ts-ignore // typescript cannot infer type this.models.handpose, - // @ts-ignore false warning with latest @typescript-eslint + // @ts-ignore // typescript cannot infer type this.models.posenet, this.models.blazepose, this.models.efficientpose, @@ -422,15 +422,14 @@ export class Human { if (this.config.async) { [faceRes, bodyRes, handRes, objectRes] = await Promise.all([faceRes, bodyRes, handRes, objectRes]); } - process.tensor.dispose(); + tf.dispose(process.tensor); if (this.config.scoped) this.tf.engine().endScope(); this.analyze('End Scope:'); - let gestureRes = []; + let gestureRes: any[] = []; if (this.config.gesture.enabled) { timeStamp = now(); - // @ts-ignore gestureRes = [...gesture.face(faceRes), ...gesture.body(bodyRes), ...gesture.hand(handRes), ...gesture.iris(faceRes)]; if (!this.config.async) this.perf.gesture = Math.trunc(now() - timeStamp); else if (this.perf.gesture) delete this.perf.gesture; @@ -507,8 +506,8 @@ export class Human { #warmupNode = async () => { const atob = (str) => Buffer.from(str, 'base64'); const img = this.config.warmup === 'face' ? atob(sample.face) : atob(sample.body); - // @ts-ignore - const data = tf.node.decodeJpeg(img); // tf.node is only defined when compiling for nodejs + // @ts-ignore // tf.node is only defined when compiling for nodejs + const data = tf.node?.decodeJpeg(img); const expanded = data.expandDims(0); this.tf.dispose(data); // log('Input:', expanded); diff --git a/src/image/image.ts b/src/image/image.ts index 6031ac77..58bd5bcc 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -1,19 +1,17 @@ -// @ts-nocheck - import * as tf from '../../dist/tfjs.esm.js'; import * as fxImage from './imagefx'; const maxSize = 2048; // internal temp canvases -let inCanvas = null; -let outCanvas = null; +let inCanvas; +let outCanvas; // instance of fximage -let fx = null; +let fx; // process input image and return tensor // input can be tensor, imagedata, htmlimageelement, htmlvideoelement // input is resized and run through imagefx filter -export function process(input, config): { tensor: tf.Tensor, canvas: OffscreenCanvas | HTMLCanvasElement } { +export function process(input, config): { tensor: typeof tf.Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement } { let tensor; if (!input) throw new Error('Human: Input is missing'); if ( @@ -32,8 +30,8 @@ export function process(input, config): { tensor: tf.Tensor, canvas: OffscreenCa if (input instanceof tf.Tensor) { tensor = tf.clone(input); } else { - const originalWidth = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); - const originalHeight = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0)); + const originalWidth = input['naturalWidth'] || input['videoWidth'] || input['width'] || (input['shape'] && (input['shape'][1] > 0)); + const originalHeight = input['naturalHeight'] || input['videoHeight'] || input['height'] || (input['shape'] && (input['shape'][2] > 0)); let targetWidth = originalWidth; let targetHeight = originalHeight; if (targetWidth > maxSize) { @@ -49,19 +47,19 @@ export function process(input, config): { tensor: tf.Tensor, canvas: OffscreenCa if (config.filter.height > 0) targetHeight = config.filter.height; else if (config.filter.width > 0) targetHeight = originalHeight * (config.filter.width / originalWidth); if (!targetWidth || !targetHeight) throw new Error('Human: Input cannot determine dimension'); - if (!inCanvas || (inCanvas.width !== targetWidth) || (inCanvas.height !== targetHeight)) { + if (!inCanvas || (inCanvas?.width !== targetWidth) || (inCanvas?.height !== targetHeight)) { inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas'); - if (inCanvas.width !== targetWidth) inCanvas.width = targetWidth; - if (inCanvas.height !== targetHeight) inCanvas.height = targetHeight; + if (inCanvas?.width !== targetWidth) inCanvas.width = targetWidth; + if (inCanvas?.height !== targetHeight) inCanvas.height = targetHeight; } const ctx = inCanvas.getContext('2d'); if (input instanceof ImageData) ctx.putImageData(input, 0, 0); - else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, inCanvas.width, inCanvas.height); + else ctx.drawImage(input, 0, 0, originalWidth, originalHeight, 0, 0, inCanvas?.width, inCanvas?.height); if (config.filter.enabled) { - if (!fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas.height !== outCanvas.height)) { - outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(inCanvas.width, inCanvas.height) : document.createElement('canvas'); - if (outCanvas.width !== inCanvas.width) outCanvas.width = inCanvas.width; - if (outCanvas.height !== inCanvas.height) outCanvas.height = inCanvas.height; + if (!fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas?.height !== outCanvas?.height)) { + outCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(inCanvas?.width, inCanvas?.height) : document.createElement('canvas'); + if (outCanvas?.width !== inCanvas?.width) outCanvas.width = inCanvas?.width; + if (outCanvas?.height !== inCanvas?.height) outCanvas.height = inCanvas?.height; // log('created FX filter'); fx = tf.ENV.flags.IS_BROWSER ? new fxImage.GLImageFilter({ canvas: outCanvas }) : null; // && (typeof document !== 'undefined') } diff --git a/src/image/imagefx.js b/src/image/imagefx.js index d12d19f4..65d55751 100644 --- a/src/image/imagefx.js +++ b/src/image/imagefx.js @@ -17,10 +17,7 @@ function GLProgram(gl, vertexSource, fragmentSource) { const shader = gl.createShader(type); gl.shaderSource(shader, source); gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - // @ts-ignore - throw new Error('Filter: GL compile failed', gl.getShaderInfoLog(shader)); - } + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error('Filter: GL compile failed', gl.getShaderInfoLog(shader)); return shader; }; @@ -33,10 +30,7 @@ function GLProgram(gl, vertexSource, fragmentSource) { gl.attachShader(this.id, _fsh); gl.linkProgram(this.id); - if (!gl.getProgramParameter(this.id, gl.LINK_STATUS)) { - // @ts-ignore - throw new Error('Filter: GL link failed', gl.getProgramInfoLog(this.id)); - } + if (!gl.getProgramParameter(this.id, gl.LINK_STATUS)) throw new Error('Filter: GL link failed', gl.getProgramInfoLog(this.id)); gl.useProgram(this.id); // Collect attributes @@ -123,7 +117,6 @@ export function GLImageFilter(params) { }; const _getTempFramebuffer = function (index) { - // @ts-ignore _tempFramebuffers[index] = _tempFramebuffers[index] || _createFramebufferTexture(_width, _height); return _tempFramebuffers[index]; }; @@ -138,7 +131,6 @@ export function GLImageFilter(params) { source = _sourceTexture; } else { // All following draw calls use the temp buffer last drawn to - // @ts-ignore source = _getTempFramebuffer(_currentFramebufferIndex)?.texture; } _drawCount++; @@ -151,7 +143,6 @@ export function GLImageFilter(params) { } else { // Intermediate draw call - get a temp buffer to draw to _currentFramebufferIndex = (_currentFramebufferIndex + 1) % 2; - // @ts-ignore target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo; } // Bind the source and target and draw the two triangles diff --git a/src/nanodet/nanodet.ts b/src/nanodet/nanodet.ts index 2355fc68..60272af9 100644 --- a/src/nanodet/nanodet.ts +++ b/src/nanodet/nanodet.ts @@ -12,8 +12,9 @@ const scaleBox = 2.5; // increase box size export async function load(config) { if (!model) { model = await tf.loadGraphModel(join(config.modelBasePath, config.object.modelPath)); - // @ts-ignore - model.inputSize = parseInt(Object.values(model.modelSignature['inputs'])[0].tensorShape.dim[2].size); + const inputs = Object.values(model.modelSignature['inputs']); + model.inputSize = Array.isArray(inputs) ? parseInt(inputs[0].tensorShape.dim[2].size) : null; + if (!model.inputSize) throw new Error(`Human: Cannot determine model inputSize: ${config.object.modelPath}`); if (!model || !model.modelUrl) log('load model failed:', config.object.modelPath); else if (config.debug) log('load model:', model.modelUrl); } diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index aba6084f..596b9278 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -4,8 +4,8 @@ import * as tf from '../../dist/tfjs.esm.js'; export const config = { name: 'humangl', priority: 99, - canvas: null, - gl: null, + canvas: null, + gl: null, width: 1024, height: 1024, webGLattr: { // https://www.khronos.org/registry/webgl/specs/latest/1.0/#5.2 @@ -24,14 +24,12 @@ export function register(): void { if (!tf.findBackend(config.name)) { log('backend registration:', config.name); try { - // @ts-ignore config.canvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(config.width, config.height) : document.createElement('canvas'); } catch (err) { log('error: cannot create canvas:', err); return; } try { - // @ts-ignore config.gl = config.canvas.getContext('webgl2', config.webGLattr); } catch (err) { log('error: cannot get WebGL2 context:', err); @@ -62,7 +60,6 @@ export function register(): void { } try { tf.ENV.set('WEBGL_VERSION', 2); - // @ts-ignore // tf.ENV.set('WEBGL_MAX_TEXTURE_SIZE', config.gl.getParameter(config.gl.MAX_TEXTURE_SIZE)); // tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true); diff --git a/wiki b/wiki index d09d36cf..652dee10 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d09d36cf9de309728504402bf301ac3ab018de65 +Subproject commit 652dee1013ae99789199cc229d6652b3323ae7de