ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add platform and backend capabilities detection

pull/356/head
Vladimir Mandic 2021-09-13 23:24:04 -04:00
parent f4caef2e90
commit 6eaea226da
7 changed files with 90 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -300,7 +300,7 @@ Default models in Human library are:
- **Body Analysis**: MoveNet - Lightning variation
- **Hand Analysis**: MediaPipe Hands
- **Body Segmentation**: Google Selfie
- **Object Detection**: CenterNet
- **Object Detection**: MB3 CenterNet
- **Body Segmentation**: Google Selfie
Note that alternative models are provided and can be enabled via configuration

171
demo/nodejs/node-canvas.js
View File

@ -1,159 +1,68 @@
/**
* Human demo for NodeJS
* Human demo for NodeJS using Canvas library
*/
const log = require('@vladmandic/pilogger');
const fs = require('fs');
const process = require('process');
const log = require('@vladmandic/pilogger');
const canvas = require('canvas');
require('@tensorflow/tfjs-node'); // for nodejs, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
const Human = require('../../dist/human.node.js'); // this is 'const Human = require('../dist/human.node-gpu.js').default;'
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
// load specific version of Human library that matches TensorFlow mode
const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default;
let human = null;
const myConfig = {
backend: 'tensorflow',
modelBasePath: 'file://models/',
const config = { // just enable all and leave default settings
debug: false,
async: true,
filter: { enabled: false },
face: {
enabled: true,
detector: { enabled: true },
mesh: { enabled: true },
iris: { enabled: true },
description: { enabled: true },
emotion: { enabled: true },
},
face: { enabled: true }, // includes mesh, iris, emotion, descriptor
hand: { enabled: true },
body: { enabled: true },
object: { enabled: true },
gestures: { enabled: true },
};
async function init() {
// create instance of human
human = new Human(myConfig);
// wait until tf is ready
await human.tf.ready();
// pre-load models
async function main() {
log.header();
// init
const human = new Human.Human(config); // create instance of human
log.info('Human:', human.version);
await human.load();
const loaded = Object.keys(human.models).filter((a) => human.models[a]);
log.info('Loaded:', loaded);
// @ts-ignore
human.env.Canvas = canvas.Canvas; // monkey-patch human to use external canvas library
await human.load(); // pre-load models
log.info('Loaded models:', Object.keys(human.models).filter((a) => human.models[a]));
log.info('Memory state:', human.tf.engine().memory());
}
async function detect(input, output) {
// read input image from file or url into buffer
let buffer;
log.info('Loading image:', input);
if (input.startsWith('http:') || input.startsWith('https:')) {
const fetch = (await import('node-fetch')).default;
const res = await fetch(input);
if (res && res.ok) buffer = await res.buffer();
else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
} else {
buffer = fs.readFileSync(input);
}
if (!buffer) return {};
// parse cmdline
const input = process.argv[2];
const output = process.argv[3];
if (process.argv.length !== 4) log.error('Parameters: <input-image> <output-image> missing');
else if (!fs.existsSync(input) && !input.startsWith('http')) log.error(`File not found: ${process.argv[2]}`);
else {
// everything seems ok
const inputImage = await canvas.loadImage(input); // load image using canvas library
log.info('Loaded image', input, inputImage.width, inputImage.height);
const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas
const ctx = inputCanvas.getContext('2d');
ctx.drawImage(inputImage, 0, 0); // draw input image onto canvas
// decode image using tfjs-node so we don't need external depenencies
/*
const tensor = human.tf.tidy(() => {
const decode = human.tf.node.decodeImage(buffer, 3);
let expand;
if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
const channels = human.tf.split(decode, 4, 2); // split rgba to channels
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
} else {
expand = human.tf.expandDims(decode, 0);
}
const cast = human.tf.cast(expand, 'float32');
return cast;
});
*/
// run detection
const result = await human.detect(inputCanvas);
// decode image using canvas library
const inputImage = await canvas.loadImage(input);
const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image');
const inputCtx = inputCanvas.getContext('2d');
inputCtx.drawImage(inputImage, 0, 0);
const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height);
const tensor = human.tf.tidy(() => {
const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]);
const channels = human.tf.split(data, 4, 2); // split rgba to channels
const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
const cast = human.tf.cast(expand, 'float32');
return cast;
});
// image shape contains image dimensions and depth
log.state('Processing:', tensor['shape']);
// run actual detection
let result;
try {
result = await human.detect(tensor, myConfig);
} catch (err) {
log.error('caught');
}
// dispose image tensor as we no longer need it
human.tf.dispose(tensor);
// print data to console
if (result) {
// invoke persons getter
const persons = result.persons;
log.data('Detected:');
// print results summary
const persons = result.persons; // invoke persons getter, only used to print summary on console
for (let i = 0; i < persons.length; i++) {
const face = persons[i].face;
const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
const body = persons[i].body;
const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
log.data(`Detected: #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
}
}
// load and draw original image
const outputCanvas = new canvas.Canvas(tensor.shape[2], tensor.shape[1], 'image'); // decoded tensor shape tells us width and height
const ctx = outputCanvas.getContext('2d');
const original = await canvas.loadImage(buffer); // we already have input as buffer, so lets reuse it
ctx.drawImage(original, 0, 0, outputCanvas.width, outputCanvas.height); // draw original to new canvas
// draw human results on canvas
// human.setCanvas(outputCanvas); // tell human to use this canvas
human.draw.all(outputCanvas, result); // human will draw results as overlays on canvas
// write canvas to new image file
const out = fs.createWriteStream(output);
out.on('finish', () => log.state('Created output image:', output));
out.on('error', (err) => log.error('Error creating image:', output, err));
const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
stream.pipe(out);
return result;
}
async function main() {
log.header();
log.info('Current folder:', process.env.PWD);
await init();
const input = process.argv[2];
const output = process.argv[3];
if (process.argv.length !== 4) {
log.error('Parameters: <input-image> <output-image> missing');
} else if (!fs.existsSync(input) && !input.startsWith('http')) {
log.error(`File not found: ${process.argv[2]}`);
} else {
await detect(input, output);
// draw detected results onto canvas and save it to a file
human.draw.all(inputCanvas, result); // use human build-in method to draw results as overlays on canvas
const outFile = fs.createWriteStream(output); // write canvas to new image file
outFile.on('finish', () => log.state('Output image:', output, inputCanvas.width, inputCanvas.height));
outFile.on('error', (err) => log.error('Output error:', output, err));
const stream = inputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
stream.pipe(outFile);
}
}

68
src/env.ts
View File

@ -1,4 +1,5 @@
import * as tf from '../dist/tfjs.esm.js';
import * as image from './image/image';
export interface Env {
browser: undefined | boolean,
@ -9,20 +10,22 @@ export interface Env {
backends: string[],
tfjs: {
version: undefined | string,
external: undefined | boolean,
},
wasm: {
supported: undefined | boolean,
backend: undefined | boolean,
simd: undefined | boolean,
multithread: undefined | boolean,
},
webgl: {
supported: undefined | boolean,
backend: undefined | boolean,
version: undefined | string,
renderer: undefined | string,
},
webgpu: {
supported: undefined | boolean,
backend: undefined | boolean,
adapter: undefined | string,
},
kernels: string[],
@ -39,20 +42,22 @@ export const env: Env = {
backends: [],
tfjs: {
version: undefined,
external: undefined,
},
wasm: {
supported: undefined,
backend: undefined,
simd: undefined,
multithread: undefined,
},
webgl: {
supported: undefined,
backend: undefined,
version: undefined,
renderer: undefined,
},
webgpu: {
supported: undefined,
backend: undefined,
adapter: undefined,
},
kernels: [],
@ -60,7 +65,7 @@ export const env: Env = {
Image: undefined,
};
export function cpuinfo() {
export async function cpuInfo() {
const cpu = { model: '', flags: [] };
if (env.node && env.platform?.startsWith('linux')) {
// eslint-disable-next-line global-require
@ -81,6 +86,37 @@ export function cpuinfo() {
else env['cpu'] = cpu;
}
export async function backendInfo() {
// analyze backends
env.backends = Object.keys(tf.engine().registryFactory);
env.wasm.supported = typeof WebAssembly !== 'undefined';
env.wasm.backend = env.backends.includes('wasm');
if (env.wasm.supported && env.wasm.backend) {
env.wasm.simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
env.wasm.multithread = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
}
const c = image.canvas(100, 100);
const ctx = c ? c.getContext('webgl2') : undefined;
env.webgl.supported = typeof ctx !== 'undefined';
env.webgl.backend = env.backends.includes('webgl');
if (env.webgl.supported && env.webgl.backend) {
// @ts-ignore getGPGPUContext only exists on WebGL backend
const gl = tf.backend().gpgpu !== 'undefined' ? await tf.backend().getGPGPUContext().gl : null;
if (gl) {
env.webgl.version = gl.getParameter(gl.VERSION);
env.webgl.renderer = gl.getParameter(gl.RENDERER);
}
}
env.webgpu.supported = env.browser && typeof navigator['gpu'] !== 'undefined';
env.webgpu.backend = env.backends.includes('webgpu');
if (env.webgpu.supported) env.webgpu.adapter = (await navigator['gpu'].requestAdapter())?.name;
// enumerate kernels
env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase());
}
export async function get() {
env.browser = typeof navigator !== 'undefined';
env.node = typeof process !== 'undefined';
@ -103,30 +139,8 @@ export async function get() {
env.agent = `NodeJS ${process.version}`;
}
// analyze backends
env.backends = Object.keys(tf.engine().registryFactory);
env.wasm.supported = env.backends.includes('wasm');
if (env.wasm.supported) {
env.wasm.simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
env.wasm.multithread = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
}
env.webgl.supported = typeof tf.backend().gpgpu !== 'undefined';
if (env.webgl.supported) {
// @ts-ignore getGPGPUContext only exists on WebGL backend
const gl = await tf.backend().getGPGPUContext().gl;
if (gl) {
env.webgl.version = gl.getParameter(gl.VERSION);
env.webgl.renderer = gl.getParameter(gl.RENDERER);
}
}
env.webgpu.supported = env.browser && typeof navigator['gpu'] !== 'undefined';
if (env.webgpu.supported) env.webgpu.adapter = (await navigator['gpu'].requestAdapter())?.name;
// enumerate kernels
env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase());
await backendInfo();
// get cpu info
// cpuinfo();
// await cpuInfo();
}

5
src/image/image.ts
View File

@ -29,10 +29,9 @@ export function canvas(width, height) {
}
} else {
// @ts-ignore // env.canvas is an external monkey-patch
// eslint-disable-next-line new-cap
c = (typeof env.Canvas !== 'undefined') ? new env.Canvas(width, height) : null;
}
if (!c) throw new Error('Human: Cannot create canvas');
// if (!c) throw new Error('Human: Cannot create canvas');
return c;
}
@ -100,7 +99,7 @@ export function process(input: Input, config: Config): { tensor: Tensor | null,
}
}
// imagefx transforms using gl
if (config.filter.enabled) {
if (config.filter.enabled && env.webgl.supported) {
if (!fx || !outCanvas || (inCanvas.width !== outCanvas.width) || (inCanvas?.height !== outCanvas?.height)) {
outCanvas = canvas(inCanvas?.width, inCanvas?.height);
if (outCanvas?.width !== inCanvas?.width) outCanvas.width = inCanvas?.width;

5
src/tfjs/humangl.ts
View File

@ -5,6 +5,7 @@
import { log } from '../helpers';
import * as tf from '../../dist/tfjs.esm.js';
import * as image from '../image/image';
export const config = {
name: 'humangl',
@ -46,13 +47,13 @@ export function register(): void {
if (!tf.findBackend(config.name)) {
// log('backend registration:', config.name);
try {
config.canvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(config.width, config.height) : document.createElement('canvas');
config.canvas = image.canvas(100, 100);
} catch (err) {
log('error: cannot create canvas:', err);
return;
}
try {
config.gl = config.canvas.getContext('webgl2', config.webGLattr) as WebGL2RenderingContext;
config.gl = config.canvas?.getContext('webgl2', config.webGLattr) as WebGL2RenderingContext;
} catch (err) {
log('error: cannot get WebGL2 context:', err);
return;

2
test/test-node-wasm.js
View File

@ -4,7 +4,9 @@ const { Canvas, Image } = require('canvas');
const Human = require('../dist/human.node-wasm.js');
const test = require('./test-main.js').test;
// @ts-ignore
Human.env.Canvas = Canvas;
// @ts-ignore
Human.env.Image = Image;
const config = {

2
wiki

@ -1 +1 @@
Subproject commit 0e902fcb57bdf9b65ed5e7ef281a699e95db6d99
Subproject commit 44b1bf12ab5dbf4cedde34da123237b1cd02627b