diff --git a/README.md b/README.md index 892de70d..745adbe4 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ **Package**: **Live Demo**: +Compatible with Browser, WebWorker and NodeJS** execution! + *Suggestions are welcome!*
@@ -88,13 +90,15 @@ You also need to install and includ `tfjs` in your project Install with: ```shell - npm install @tensorflow/tfjs @vladmandic/human + npm install @tensorflow/tfjs-node @vladmandic/human ``` And then use with: ```js - import * as tf from '@tensorflow/tfjs'; - import human from '@vladmandic/Human'; + const tf = require('@tensorflow/tfjs-node'); + const human = require('@vladmandic/human'); ``` +*See limitations for NodeJS usage under `demo`* + ### Weights @@ -108,10 +112,20 @@ If your application resides in a different folder, modify `modelPath` property i Demos are included in `/demo`: -- `demo-esm`: Demo using ESM module -- `demo-iife`: Demo using IIFE module +Browser: +- `demo-esm`: Demo using Browser with ESM module +- `demo-iife`: Demo using Browser with IIFE module +- `demo-webworker`: Demo using Browser with ESM module and Web Workers +*All three following demos are identical, they just illustrate different ways to load and work with `Human` library:* -Both demos are identical, they just illustrate different ways to load `Human` library +NodeJS: +- `demo-node`: Demo using NodeJS with CJS module + This is a very simple demo as althought `Human` library is compatible with NodeJS execution + and is able to load images and models from local filesystem, + `tfjs-node` backend does not implement function required for execution of some models + + Currently only body pose detection works while face and hand models are not supported + See `tfjs-node` issue for details
@@ -219,34 +233,34 @@ Result of `humand.detect()` is a single object that includes data for all enable ```js result = { - face: // + face: // [ { - confidence: // - box: // - mesh: // (468 base points & 10 iris points) - annotations: // (32 base annotated landmarks & 2 iris annotations) - iris: // (relative distance of iris to camera, multiple by focal lenght to get actual distance) - age: // (estimated age) - gender: // (male or female) + confidence, // + box, // + mesh, // (468 base points & 10 iris points) + annotations, // (32 base annotated landmarks & 2 iris annotations) + iris, // (relative distance of iris to camera, multiple by focal lenght to get actual distance) + age, // (estimated age) + gender, // (male or female) } ], - body: // + body: // [ { - score: // , - keypoints: // (17 annotated landmarks) + score, // , + keypoints, // (17 annotated landmarks) } ], hand: // [ { - confidence: // , - box: // , - landmarks: // (21 points) - annotations: // ]> (5 annotated landmakrs) + confidence, // , + box, // , + landmarks, // (21 points) + annotations, // ]> (5 annotated landmakrs) } - ] + ], } ``` @@ -286,4 +300,6 @@ Library can also be used on mobile devices ## Todo - Improve detection of smaller faces +- Tweak default parameters - Verify age/gender models +- Make it work with multiple hands diff --git a/demo/demo-esm.js b/demo/demo-esm.js index 3e9b4a9a..8a3b2e1e 100644 --- a/demo/demo-esm.js +++ b/demo/demo-esm.js @@ -215,9 +215,11 @@ function setupGUI() { settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); settings.addRange('FPS', 0, 100, 0, 1); settings.addBoolean('Pause', false, (val) => { - if (val) document.getElementById('video').pause(); - else document.getElementById('video').play(); - runHumanDetect(); + const video = document.getElementById('video'); + const canvas = document.getElementById('canvas'); + if (val) video.pause(); + else video.play(); + runHumanDetect(video, canvas); }); settings.addHTML('line1', '
'); settings.hideTitle('line1'); settings.addBoolean('Draw Boxes', false); @@ -283,10 +285,10 @@ async function setupCamera() { video.srcObject = stream; return new Promise((resolve) => { video.onloadedmetadata = () => { - resolve(video); video.width = video.videoWidth; video.height = video.videoHeight; video.play(); + resolve(video); }; }); } diff --git a/demo/demo-node.js b/demo/demo-node.js new file mode 100644 index 00000000..26dead65 --- /dev/null +++ b/demo/demo-node.js @@ -0,0 +1,68 @@ +const fs = require('fs'); +const process = require('process'); +const console = require('console'); +const tf = require('@tensorflow/tfjs-node'); +const human = require('..'); // this would be '@vladmandic/human' + +const logger = new console.Console({ + stdout: process.stdout, + stderr: process.stderr, + ignoreErrors: true, + groupIndentation: 2, + inspectOptions: { + showHidden: true, + depth: 5, + colors: true, + showProxy: true, + maxArrayLength: 1024, + maxStringLength: 10240, + breakLength: 200, + compact: 64, + sorted: false, + getters: true, + }, +}); + +const config = { + face: { + enabled: false, + detector: { modelPath: 'file://models/blazeface/model.json', inputSize: 128, maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 }, + mesh: { enabled: true, modelPath: 'file://models/facemesh/model.json', inputSize: 192 }, + iris: { enabled: true, modelPath: 'file://models/iris/model.json', inputSize: 192 }, + age: { enabled: true, modelPath: 'file://models/ssrnet-age/imdb/model.json', inputSize: 64, skipFrames: 5 }, + gender: { enabled: true, modelPath: 'file://models/ssrnet-gender/imdb/model.json' }, + }, + body: { enabled: true, modelPath: 'file://models/posenet/model.json', inputResolution: 257, outputStride: 16, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 }, + hand: { + enabled: false, + inputSize: 256, + skipFrames: 5, + minConfidence: 0.8, + iouThreshold: 0.3, + scoreThreshold: 0.75, + detector: { anchors: 'file://models/handdetect/anchors.json', modelPath: 'file://models/handdetect/model.json' }, + skeleton: { modelPath: 'file://models/handskeleton/model.json' }, + }, +}; + +async function detect(input, output) { + await tf.setBackend('tensorflow'); + await tf.ready(); + logger.info('TFJS Flags:', tf.env().features); + logger.log('Loading:', input); + const buffer = fs.readFileSync(input); + const image = tf.node.decodeImage(buffer); + logger.log('Processing:', image.shape); + const result = await human.detect(image, config); + logger.log(result); + // Draw detected data and save processed image + logger.log('Saving:', output); +} + +async function main() { + if (process.argv.length !== 4) logger.error('Parameters: '); + else if (!fs.existsSync(process.argv[2])) logger.error(`File not found: ${process.argv[2]}`); + else detect(process.argv[2], process.argv[3]); +} + +main(); diff --git a/demo/demo-webworker-worker.js b/demo/demo-webworker-worker.js new file mode 100644 index 00000000..0e3a7eb4 --- /dev/null +++ b/demo/demo-webworker-worker.js @@ -0,0 +1,22 @@ +import human from '../dist/human.esm.js'; + +onmessage = async (msg) => { + const result = await human.detect(msg.data.image, msg.data.config); + postMessage(result); +}; + +/* + +web workers are finicky +- cannot pass HTMLImage or HTMLVideo to web worker, so need to pass canvas instead +- canvases can execute transferControlToOffscreen() and then become offscreenCanvas which can be passed to worker, but... + cannot transfer canvas that has a rendering context (basically, first time you execute getContext() on it) + +which means that if we pass main Canvas that will be used to render results on, +then all operations on it must be within webworker and we cannot touch it in the main thread at all. +doable, but...how to paint a video frame on it before we pass it? + +and we create new offscreenCanvas that we drew video frame on and pass it's imageData and return results from worker +then there is an overhead of creating it and it ends up being slower than executing in the main thread + +*/ diff --git a/package.json b/package.json index 5acb6919..4341c6d4 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.2.8", "description": "human: 3D Face Detection, Iris Tracking and Age & Gender Prediction", "sideEffects": false, - "main": "src/index.js", + "main": "dist/human.node.js", "module": "dist/human.esm.js", "browser": "dist/human.js", "author": "Vladimir Mandic ", @@ -20,7 +20,8 @@ "url": "git+https://github.com/vladmandic/human.git" }, "dependencies": { - "@tensorflow/tfjs": "^2.6.0" + "@tensorflow/tfjs": "^2.6.0", + "@tensorflow/tfjs-node": "^2.6.0" }, "devDependencies": { "esbuild": "^0.7.13", @@ -33,10 +34,12 @@ "rimraf": "^3.0.2" }, "scripts": { - "lint": "eslint src/*", - "build": "rimraf dist/ && npm run build-esm && npm run build-iife", - "build-esm": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --outfile=dist/human.esm.js src/index.js", - "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --global-name=human --outfile=dist/human.js src/index.js" + "start": "node --trace-warnings --trace-uncaught --no-deprecation demo/demo-node.js", + "lint": "eslint src/*.js demo/*.js", + "build": "rimraf dist/ && npm run build-esm && npm run build-iife && npm run build-node", + "build-esm": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=esm --minify --external:fs --outfile=dist/human.esm.js src/index.js", + "build-iife": "esbuild --bundle --platform=browser --sourcemap --target=esnext --format=iife --minify --external:fs --global-name=human --outfile=dist/human.js src/index.js", + "build-node": "esbuild --bundle --platform=node --sourcemap --target=esnext --format=cjs --external:@tensorflow --outfile=dist/human.node.js src/index.js" }, "keywords": [ "tensorflowjs", diff --git a/src/handpose/index.js b/src/handpose/index.js index b4f42ad7..5f099816 100644 --- a/src/handpose/index.js +++ b/src/handpose/index.js @@ -17,9 +17,13 @@ async function loadHandPoseModel(url) { // of bounding boxes, each of which is assigned a score during prediction. The // anchors define the coordinates of these boxes. async function loadAnchors(url) { - return tf.util - .fetch(url) - .then((d) => d.json()); + if (tf.env().features.IS_NODE) { + // eslint-disable-next-line global-require + const fs = require('fs'); + const data = await fs.readFileSync(url.replace('file://', '')); + return JSON.parse(data); + } + return tf.util.fetch(url).then((d) => d.json()); } /** diff --git a/src/index.js b/src/index.js index a367b38a..dd190674 100644 --- a/src/index.js +++ b/src/index.js @@ -31,53 +31,55 @@ function mergeDeep(...objects) { } async function detect(input, userConfig) { - const config = mergeDeep(defaults, userConfig); + // eslint-disable-next-line no-async-promise-executor + return new Promise(async (resolve) => { + const config = mergeDeep(defaults, userConfig); - // load models if enabled - if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body); - if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand); - if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face); - if (config.face.age.enabled) await ssrnet.loadAge(config); - if (config.face.gender.enabled) await ssrnet.loadGender(config); + // load models if enabled + if (config.face.age.enabled) await ssrnet.loadAge(config); + if (config.face.gender.enabled) await ssrnet.loadGender(config); + if (config.body.enabled && !models.posenet) models.posenet = await posenet.load(config.body); + if (config.hand.enabled && !models.handpose) models.handpose = await handpose.load(config.hand); + if (config.face.enabled && !models.facemesh) models.facemesh = await facemesh.load(config.face); - tf.engine().startScope(); + tf.engine().startScope(); - // run posenet - let poseRes = []; - if (config.body.enabled) poseRes = await models.posenet.estimateMultiplePoses(input, config.body); + // run posenet + let poseRes = []; + if (config.body.enabled) poseRes = await models.posenet.estimateMultiplePoses(input, config.body); - // run handpose - let handRes = []; - if (config.hand.enabled) handRes = await models.handpose.estimateHands(input, config.hand); + // run handpose + let handRes = []; + if (config.hand.enabled) handRes = await models.handpose.estimateHands(input, config.hand); - // run facemesh, includes blazeface and iris - const faceRes = []; - if (config.face.enabled) { - const faces = await models.facemesh.estimateFaces(input, config.face); - for (const face of faces) { - // run ssr-net age & gender, inherits face from blazeface - const ssrdata = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {}; - face.image.dispose(); - // iris: array[ bottom, left, top, right, center ] - const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris) - ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) - : 0; - faceRes.push({ - confidence: face.confidence, - box: face.box, - mesh: face.mesh, - annotations: face.annotations, - age: ssrdata.age, - gender: ssrdata.gender, - iris: (iris !== 0) ? Math.trunc(100 * 11.7 / iris) / 100 : 0, - }); + // run facemesh, includes blazeface and iris + const faceRes = []; + if (config.face.enabled) { + const faces = await models.facemesh.estimateFaces(input, config.face); + for (const face of faces) { + // run ssr-net age & gender, inherits face from blazeface + const ssrdata = (config.face.age.enabled || config.face.gender.enabled) ? await ssrnet.predict(face.image, config) : {}; + face.image.dispose(); + // iris: array[ bottom, left, top, right, center ] + const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris) + ? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) + : 0; + faceRes.push({ + confidence: face.confidence, + box: face.box, + mesh: face.mesh, + annotations: face.annotations, + age: ssrdata.age, + gender: ssrdata.gender, + iris: (iris !== 0) ? Math.trunc(100 * 11.7 / iris) / 100 : 0, + }); + } } - } - tf.engine().endScope(); - - // combine results - return { face: faceRes, body: poseRes, hand: handRes }; + tf.engine().endScope(); + // combine results + resolve({ face: faceRes, body: poseRes, hand: handRes }); + }); } exports.detect = detect;