diff --git a/README.md b/README.md index f9f4c948..8ff7ba16 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,7 @@ Default models in Human library are: - **Body Analysis**: MoveNet - Lightning variation - **Hand Analysis**: MediaPipe Hands - **Body Segmentation**: Google Selfie -- **Object Detection**: CenterNet +- **Object Detection**: MB3 CenterNet - **Body Segmentation**: Google Selfie Note that alternative models are provided and can be enabled via configuration diff --git a/demo/nodejs/node-canvas.js b/demo/nodejs/node-canvas.js index 8b80c42f..72469477 100644 --- a/demo/nodejs/node-canvas.js +++ b/demo/nodejs/node-canvas.js @@ -1,159 +1,68 @@ /** - * Human demo for NodeJS + * Human demo for NodeJS using Canvas library */ -const log = require('@vladmandic/pilogger'); const fs = require('fs'); const process = require('process'); +const log = require('@vladmandic/pilogger'); const canvas = require('canvas'); +require('@tensorflow/tfjs-node'); // for nodejs, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human +const Human = require('../../dist/human.node.js'); // this is 'const Human = require('../dist/human.node-gpu.js').default;' -// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human -// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu'); - -// load specific version of Human library that matches TensorFlow mode -const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default; - -let human = null; - -const myConfig = { - backend: 'tensorflow', - modelBasePath: 'file://models/', +const config = { // just enable all and leave default settings debug: false, - async: true, - filter: { enabled: false }, - face: { - enabled: true, - detector: { enabled: true }, - mesh: { enabled: true }, - iris: { enabled: true }, - description: { enabled: true }, - emotion: { enabled: true }, - }, + face: { enabled: true }, // includes mesh, iris, emotion, descriptor hand: { enabled: true }, body: { enabled: true }, object: { enabled: true }, + gestures: { enabled: true }, }; -async function init() { - // create instance of human - human = new Human(myConfig); - // wait until tf is ready - await human.tf.ready(); - // pre-load models +async function main() { + log.header(); + + // init + const human = new Human.Human(config); // create instance of human log.info('Human:', human.version); - await human.load(); - const loaded = Object.keys(human.models).filter((a) => human.models[a]); - log.info('Loaded:', loaded); + // @ts-ignore + human.env.Canvas = canvas.Canvas; // monkey-patch human to use external canvas library + await human.load(); // pre-load models + log.info('Loaded models:', Object.keys(human.models).filter((a) => human.models[a])); log.info('Memory state:', human.tf.engine().memory()); -} -async function detect(input, output) { - // read input image from file or url into buffer - let buffer; - log.info('Loading image:', input); - if (input.startsWith('http:') || input.startsWith('https:')) { - const fetch = (await import('node-fetch')).default; - const res = await fetch(input); - if (res && res.ok) buffer = await res.buffer(); - else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type')); - } else { - buffer = fs.readFileSync(input); - } - if (!buffer) return {}; + // parse cmdline + const input = process.argv[2]; + const output = process.argv[3]; + if (process.argv.length !== 4) log.error('Parameters: missing'); + else if (!fs.existsSync(input) && !input.startsWith('http')) log.error(`File not found: ${process.argv[2]}`); + else { + // everything seems ok + const inputImage = await canvas.loadImage(input); // load image using canvas library + log.info('Loaded image', input, inputImage.width, inputImage.height); + const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas + const ctx = inputCanvas.getContext('2d'); + ctx.drawImage(inputImage, 0, 0); // draw input image onto canvas - // decode image using tfjs-node so we don't need external depenencies - /* - const tensor = human.tf.tidy(() => { - const decode = human.tf.node.decodeImage(buffer, 3); - let expand; - if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb - const channels = human.tf.split(decode, 4, 2); // split rgba to channels - const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha - expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead - } else { - expand = human.tf.expandDims(decode, 0); - } - const cast = human.tf.cast(expand, 'float32'); - return cast; - }); - */ + // run detection + const result = await human.detect(inputCanvas); - // decode image using canvas library - const inputImage = await canvas.loadImage(input); - const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image'); - const inputCtx = inputCanvas.getContext('2d'); - inputCtx.drawImage(inputImage, 0, 0); - const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height); - const tensor = human.tf.tidy(() => { - const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]); - const channels = human.tf.split(data, 4, 2); // split rgba to channels - const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha - const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead - const cast = human.tf.cast(expand, 'float32'); - return cast; - }); - - // image shape contains image dimensions and depth - log.state('Processing:', tensor['shape']); - - // run actual detection - let result; - try { - result = await human.detect(tensor, myConfig); - } catch (err) { - log.error('caught'); - } - - // dispose image tensor as we no longer need it - human.tf.dispose(tensor); - - // print data to console - if (result) { - // invoke persons getter - const persons = result.persons; - log.data('Detected:'); + // print results summary + const persons = result.persons; // invoke persons getter, only used to print summary on console for (let i = 0; i < persons.length; i++) { const face = persons[i].face; const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null; const body = persons[i].body; const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null; - log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`); + log.data(`Detected: #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`); } - } - // load and draw original image - const outputCanvas = new canvas.Canvas(tensor.shape[2], tensor.shape[1], 'image'); // decoded tensor shape tells us width and height - const ctx = outputCanvas.getContext('2d'); - const original = await canvas.loadImage(buffer); // we already have input as buffer, so lets reuse it - ctx.drawImage(original, 0, 0, outputCanvas.width, outputCanvas.height); // draw original to new canvas - - // draw human results on canvas - // human.setCanvas(outputCanvas); // tell human to use this canvas - human.draw.all(outputCanvas, result); // human will draw results as overlays on canvas - - // write canvas to new image file - const out = fs.createWriteStream(output); - out.on('finish', () => log.state('Created output image:', output)); - out.on('error', (err) => log.error('Error creating image:', output, err)); - const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true }); - stream.pipe(out); - - return result; -} - -async function main() { - log.header(); - log.info('Current folder:', process.env.PWD); - await init(); - const input = process.argv[2]; - const output = process.argv[3]; - if (process.argv.length !== 4) { - log.error('Parameters: missing'); - } else if (!fs.existsSync(input) && !input.startsWith('http')) { - log.error(`File not found: ${process.argv[2]}`); - } else { - await detect(input, output); + // draw detected results onto canvas and save it to a file + human.draw.all(inputCanvas, result); // use human build-in method to draw results as overlays on canvas + const outFile = fs.createWriteStream(output); // write canvas to new image file + outFile.on('finish', () => log.state('Output image:', output, inputCanvas.width, inputCanvas.height)); + outFile.on('error', (err) => log.error('Output error:', output, err)); + const stream = inputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true }); + stream.pipe(outFile); } } diff --git a/src/env.ts b/src/env.ts index ce7b3d6a..221d8c03 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,4 +1,5 @@ import * as tf from '../dist/tfjs.esm.js'; +import * as image from './image/image'; export interface Env { browser: undefined | boolean, @@ -9,20 +10,22 @@ export interface Env { backends: string[], tfjs: { version: undefined | string, - external: undefined | boolean, }, wasm: { supported: undefined | boolean, + backend: undefined | boolean, simd: undefined | boolean, multithread: undefined | boolean, }, webgl: { supported: undefined | boolean, + backend: undefined | boolean, version: undefined | string, renderer: undefined | string, }, webgpu: { supported: undefined | boolean, + backend: undefined | boolean, adapter: undefined | string, }, kernels: string[], @@ -39,20 +42,22 @@ export const env: Env = { backends: [], tfjs: { version: undefined, - external: undefined, }, wasm: { supported: undefined, + backend: undefined, simd: undefined, multithread: undefined, }, webgl: { supported: undefined, + backend: undefined, version: undefined, renderer: undefined, }, webgpu: { supported: undefined, + backend: undefined, adapter: undefined, }, kernels: [], @@ -60,7 +65,7 @@ export const env: Env = { Image: undefined, }; -export function cpuinfo() { +export async function cpuInfo() { const cpu = { model: '', flags: [] }; if (env.node && env.platform?.startsWith('linux')) { // eslint-disable-next-line global-require @@ -81,6 +86,37 @@ export function cpuinfo() { else env['cpu'] = cpu; } +export async function backendInfo() { + // analyze backends + env.backends = Object.keys(tf.engine().registryFactory); + env.wasm.supported = typeof WebAssembly !== 'undefined'; + env.wasm.backend = env.backends.includes('wasm'); + if (env.wasm.supported && env.wasm.backend) { + env.wasm.simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); + env.wasm.multithread = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT'); + } + + const c = image.canvas(100, 100); + const ctx = c ? c.getContext('webgl2') : undefined; + env.webgl.supported = typeof ctx !== 'undefined'; + env.webgl.backend = env.backends.includes('webgl'); + if (env.webgl.supported && env.webgl.backend) { + // @ts-ignore getGPGPUContext only exists on WebGL backend + const gl = tf.backend().gpgpu !== 'undefined' ? await tf.backend().getGPGPUContext().gl : null; + if (gl) { + env.webgl.version = gl.getParameter(gl.VERSION); + env.webgl.renderer = gl.getParameter(gl.RENDERER); + } + } + + env.webgpu.supported = env.browser && typeof navigator['gpu'] !== 'undefined'; + env.webgpu.backend = env.backends.includes('webgpu'); + if (env.webgpu.supported) env.webgpu.adapter = (await navigator['gpu'].requestAdapter())?.name; + + // enumerate kernels + env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase()); +} + export async function get() { env.browser = typeof navigator !== 'undefined'; env.node = typeof process !== 'undefined'; @@ -103,30 +139,8 @@ export async function get() { env.agent = `NodeJS ${process.version}`; } - // analyze backends - env.backends = Object.keys(tf.engine().registryFactory); - env.wasm.supported = env.backends.includes('wasm'); - if (env.wasm.supported) { - env.wasm.simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); - env.wasm.multithread = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT'); - } - - env.webgl.supported = typeof tf.backend().gpgpu !== 'undefined'; - if (env.webgl.supported) { - // @ts-ignore getGPGPUContext only exists on WebGL backend - const gl = await tf.backend().getGPGPUContext().gl; - if (gl) { - env.webgl.version = gl.getParameter(gl.VERSION); - env.webgl.renderer = gl.getParameter(gl.RENDERER); - } - } - - env.webgpu.supported = env.browser && typeof navigator['gpu'] !== 'undefined'; - if (env.webgpu.supported) env.webgpu.adapter = (await navigator['gpu'].requestAdapter())?.name; - - // enumerate kernels - env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase()); + await backendInfo(); // get cpu info - // cpuinfo(); + // await cpuInfo(); } diff --git a/src/image/image.ts b/src/image/image.ts index 3bf26a07..7b6e8353 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -29,10 +29,9 @@ export function canvas(width, height) { } } else { // @ts-ignore // env.canvas is an external monkey-patch - // eslint-disable-next-line new-cap c = (typeof env.Canvas !== 'undefined') ? new env.Canvas(width, height) : null; } - if (!c) throw new Error('Human: Cannot create canvas'); + // if (!c) throw new Error('Human: Cannot create canvas'); return c; } @@ -100,7 +99,7 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, } } // imagefx transforms using gl - if (config.filter.enabled) { + if (config.filter.enabled && env.webgl.supported) { if (!fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas?.height !== outCanvas?.height)) { outCanvas = canvas(inCanvas?.width, inCanvas?.height); if (outCanvas?.width !== inCanvas?.width) outCanvas.width = inCanvas?.width; diff --git a/src/tfjs/humangl.ts b/src/tfjs/humangl.ts index f5521338..b46e0c54 100644 --- a/src/tfjs/humangl.ts +++ b/src/tfjs/humangl.ts @@ -5,6 +5,7 @@ import { log } from '../helpers'; import * as tf from '../../dist/tfjs.esm.js'; +import * as image from '../image/image'; export const config = { name: 'humangl', @@ -46,13 +47,13 @@ export function register(): void { if (!tf.findBackend(config.name)) { // log('backend registration:', config.name); try { - config.canvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(config.width, config.height) : document.createElement('canvas'); + config.canvas = image.canvas(100, 100); } catch (err) { log('error: cannot create canvas:', err); return; } try { - config.gl = config.canvas.getContext('webgl2', config.webGLattr) as WebGL2RenderingContext; + config.gl = config.canvas?.getContext('webgl2', config.webGLattr) as WebGL2RenderingContext; } catch (err) { log('error: cannot get WebGL2 context:', err); return; diff --git a/test/test-node-wasm.js b/test/test-node-wasm.js index c9cc53a3..3472ed8a 100644 --- a/test/test-node-wasm.js +++ b/test/test-node-wasm.js @@ -4,7 +4,9 @@ const { Canvas, Image } = require('canvas'); const Human = require('../dist/human.node-wasm.js'); const test = require('./test-main.js').test; +// @ts-ignore Human.env.Canvas = Canvas; +// @ts-ignore Human.env.Image = Image; const config = { diff --git a/wiki b/wiki index 0e902fcb..44b1bf12 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 0e902fcb57bdf9b65ed5e7ef281a699e95db6d99 +Subproject commit 44b1bf12ab5dbf4cedde34da123237b1cd02627b