diff --git a/README.md b/README.md index 56a3971a..267550fe 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,8 @@ config = { scoped: false, // enable scoped runs // some models *may* have memory leaks, this wrapps everything in a local scope at a cost of performance // typically not needed - filter: { + videoOptimized: true, // perform additional optimizations when input is video, must be disabled for images + filter: { // note: image filters are only available in Browser environments and not in NodeJS as they require WebGL for processing enabled: true, // enable image pre-processing filters return: true, // return processed canvas imagedata in result brightness: 0, // range: -1 (darken) to 1 (lighten) @@ -432,6 +433,15 @@ For performance details, see output of `result.performance` object during runtim
+## Limitations + +`Human` library can be used in any modern Browser or NodeJS environment, but there are several items to be aware of: + +- **NodeJS**: Due to a missing feature in `tfjs-node`, only some models are available +- **Browser**: `filters` module cannot be used when using web workers + +
+ ## Credits - Face Detection: [**MediaPipe BlazeFace**](https://drive.google.com/file/d/1f39lSzU5Oq-j_OXgS67KfN5wNsoeAZ4V/view) diff --git a/config.js b/config.js index f2557433..01b59669 100644 --- a/config.js +++ b/config.js @@ -7,6 +7,7 @@ export default { scoped: false, // enable scoped runs // some models *may* have memory leaks, this wrapps everything in a local scope at a cost of performance // typically not needed + videoOptimized: true, // perform additional optimizations when input is video, must be disabled for images filter: { enabled: true, // enable image pre-processing filters return: true, // return processed canvas imagedata in result diff --git a/demo/browser.js b/demo/browser.js index dbb6c158..0a6fc3bd 100644 --- a/demo/browser.js +++ b/demo/browser.js @@ -28,6 +28,7 @@ const ui = { const config = { backend: 'webgl', // if you want to use 'wasm' backend, enable script load of tf and tf-backend-wasm in index.html filter: { enabled: true, brightness: 0, contrast: 0, sharpness: 0, blur: 0, saturation: 0, hue: 0, negative: false, sepia: false, vintage: false, kodachrome: false, technicolor: false, polaroid: false, pixelate: 0 }, + videoOptimized: true, face: { enabled: true, detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 }, diff --git a/demo/worker.js b/demo/worker.js index 94366e5a..be030858 100644 --- a/demo/worker.js +++ b/demo/worker.js @@ -1,6 +1,7 @@ import human from '../dist/human.esm.js'; let config; +let busy = false; const log = (...msg) => { // eslint-disable-next-line no-console @@ -8,6 +9,8 @@ const log = (...msg) => { }; onmessage = async (msg) => { + if (busy) return; + busy = true; // worker.postMessage({ image: image.data.buffer, width: canvas.width, height: canvas.height, config }, [image.data.buffer]); const image = new ImageData(new Uint8ClampedArray(msg.data.image), msg.data.width, msg.data.height); config = msg.data.config; @@ -19,4 +22,5 @@ onmessage = async (msg) => { log('Worker thread error:', err.message); } postMessage(result); + busy = false; }; diff --git a/src/human.js b/src/human.js index 1d0cee28..057bd59f 100644 --- a/src/human.js +++ b/src/human.js @@ -74,15 +74,6 @@ function mergeDeep(...objects) { function sanity(input) { if (!input) return 'input is not defined'; - if (!(input instanceof tf.Tensor) - || (tf.ENV.flags.IS_BROWSER - && (input instanceof ImageData || input instanceof HTMLImageElement || input instanceof HTMLCanvasElement || input instanceof HTMLVideoElement || input instanceof HTMLMediaElement))) { - const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); - if (!width || (width === 0)) return 'input is empty'; - } - if (tf.ENV.flags.IS_BROWSER && (input instanceof HTMLVideoElement || input instanceof HTMLMediaElement)) { - if (input.readyState && (input.readyState <= 2)) return 'input is not ready'; - } if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) { return 'input must be a tensor'; } @@ -127,15 +118,18 @@ function tfImage(input) { let filtered; if (tf.ENV.flags.IS_BROWSER && config.filter.enabled && !(input instanceof tf.Tensor)) { const width = input.naturalWidth || input.videoWidth || input.width || (input.shape && (input.shape[1] > 0)); - const height = input.naturalHeight || input.videoHeight || input.Height || (input.shape && (input.shape[2] > 0)); - // if (!offscreenCanvas) offscreenCanvas = new OffscreenCanvas(width, height); + const height = input.naturalHeight || input.videoHeight || input.height || (input.shape && (input.shape[2] > 0)); + if (!offscreenCanvas) offscreenCanvas = new OffscreenCanvas(width, height); + /* if (!offscreenCanvas) { offscreenCanvas = document.createElement('canvas'); offscreenCanvas.width = width; offscreenCanvas.height = height; } + */ const ctx = offscreenCanvas.getContext('2d'); - ctx.drawImage(input, 0, 0, width, height, 0, 0, offscreenCanvas.width, offscreenCanvas.height); + if (input instanceof ImageData) ctx.putImageData(input, 0, 0); + else ctx.drawImage(input, 0, 0, width, height, 0, 0, offscreenCanvas.width, offscreenCanvas.height); if (!fx) fx = new fxImage.Canvas(); else fx.reset(); fx.addFilter('brightness', config.filter.brightness); // must have at least one filter enabled @@ -173,8 +167,8 @@ async function detect(input, userConfig = {}) { let timeStamp; timeStamp = now(); - const shouldOverride = tf.ENV.flags.IS_NODE || (tf.ENV.flags.IS_BROWSER && !((input instanceof HTMLVideoElement) || (input instanceof HTMLMediaElement))); - config = mergeDeep(defaults, userConfig, shouldOverride ? override : {}); + config = mergeDeep(defaults, userConfig); + if (!config.videoOptimized) config = mergeDeep(config, override); perf.config = Math.trunc(now() - timeStamp); // sanity checks