ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch to custom tfjs for demos

pull/356/head
Vladimir Mandic 2021-10-26 15:08:05 -04:00
parent 8c941597ed
commit 81d5336498
17 changed files with 125 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.build.json
View File

@ -111,34 +111,13 @@
"sourcemap": true,
"external": ["@tensorflow", "fs", "os", "buffer", "util"]
},
{
"name": "tfjs/browser/esm/custom",
"platform": "browser",
"format": "esm",
"input": "tfjs/tf-custom.ts",
"output": "dist/tfjs.esm.js",
"sourcemap": true,
"external": ["fs", "os", "buffer", "util"]
},
{
"name": "human/browser/esm/custom",
"platform": "browser",
"format": "esm",
"input": "src/human.ts",
"output": "dist/human.custom.esm.js",
"sourcemap": true,
"minify": true,
"external": ["fs", "os", "buffer", "util"]
},
{
"name": "tfjs/browser/esm/bundle",
"platform": "browser",
"format": "esm",
"input": "tfjs/tf-browser.ts",
"output": "dist/tfjs.esm.js",
"minify": true,
"minify": false,
"sourcemap": true,
"external": ["fs", "os", "buffer", "util"]
},
@ -158,7 +137,28 @@
"format": "esm",
"input": "src/human.ts",
"output": "dist/human.esm.js",
"minify": true,
"sourcemap": true,
"external": ["fs", "os", "buffer", "util"]
},
{
"name": "tfjs/browser/esm/custom",
"platform": "browser",
"format": "esm",
"input": "tfjs/tf-custom.ts",
"output": "dist/tfjs.esm.js",
"sourcemap": false,
"external": ["fs", "os", "buffer", "util"]
},
{
"name": "human/browser/esm/custom",
"platform": "browser",
"format": "esm",
"input": "src/human.ts",
"output": "dist/human.custom.esm.js",
"sourcemap": true,
"minify": false,
"external": ["fs", "os", "buffer", "util"],
"typings": "types",
"typedoc": "typedoc"

3
.eslintrc.json
View File

@ -11,7 +11,8 @@
"ecmaVersion": 2021
},
"plugins": [
"@typescript-eslint"
"@typescript-eslint",
"html"
],
"extends": [
"airbnb-base",

2
.hintrc
View File

@ -5,7 +5,7 @@
"browserslist": [
"chrome >= 90",
"edge >= 90",
"firefox >= 90",
"firefox >= 100",
"android >= 90",
"safari >= 15"
],

10
CHANGELOG.md
View File

@ -9,11 +9,15 @@
## Changelog
### **HEAD -> main** 2021/10/25 mandic00@live.com
### **release: 2.4.1** 2021/10/25 mandic00@live.com
### **2.4.1** 2021/10/25 mandic00@live.com
### **origin/main** 2021/10/25 mandic00@live.com
- refactoring plus jsdoc comments
- increase face similarity match resolution
- time based caching
- turn on minification

6
TODO.md
View File

@ -2,11 +2,12 @@
## Work in Progress
- Switch to custom `tfjs` for main `human` ESM bundle
<br>
### Exploring
- Switch to custom `tfjs` for main `human` ESM bundle
- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
- Histogram Equalization: Regular, Adaptive, Contrast Limited
- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
@ -22,6 +23,9 @@ Experimental support only until support is officially added in Chromium
## Known Issues
- `tfjs.esm.d.ts` missing namespace `OptimizerConstructors`
- exports from `match` are marked as private
<br>
### Face Detection

10
demo/facematch/facematch.js
View File

@ -1,12 +1,12 @@
// @ts-nocheck // typescript checks disabled as this is pure javascript
// @ts-nocheck
/**
* Human demo for browsers
*
* Demo for face descriptor analysis and face simmilarity analysis
*/
import Human from '../../dist/human.esm.js';
/** @type {Human} */
import Human from '../../dist/human.custom.esm.js';
const userConfig = {
backend: 'wasm',
@ -15,7 +15,6 @@ const userConfig = {
cacheSensitivity: 0,
debug: true,
modelBasePath: '../../models/',
// wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/',
face: {
enabled: true,
detector: { rotation: true, return: true, maxDetected: 50 },
@ -165,6 +164,7 @@ async function AddFaceCanvas(index, res, fileName) {
await human.tf.browser.toPixels(res.face[i].tensor, canvas);
document.getElementById('faces').appendChild(canvas);
const ctx = canvas.getContext('2d');
if (!ctx) return false;
ctx.font = 'small-caps 0.8rem "Lato"';
ctx.fillStyle = 'rgba(255, 255, 255, 1)';
ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
@ -258,6 +258,8 @@ async function main() {
title('');
log('Ready');
human.validate(userConfig);
human.similarity([], []);
}
window.onload = main;

2
demo/helpers/gl-bench.js
View File

@ -37,7 +37,7 @@ const UISVG = `
class GLBench {
/** GLBench constructor
* @param { WebGLRenderingContext | WebGL2RenderingContext } gl context
* @param { WebGLRenderingContext | WebGL2RenderingContext | null } gl context
* @param { Object | undefined } settings additional settings
*/
constructor(gl, settings = {}) {

31
demo/helpers/menu.js
View File

@ -1,5 +1,3 @@
//@ts-nocheck
let instance = 0;
let CSScreated = false;
@ -86,6 +84,7 @@ class Menu {
}
createMenu(parent, title = '', position = { top: null, left: null, bottom: null, right: null }) {
/** @type {HTMLDivElement} */
this.menu = document.createElement('div');
this.menu.id = `menu-${instance}`;
this.menu.className = 'menu';
@ -120,6 +119,7 @@ class Menu {
this.menu.appendChild(this.container);
if (typeof parent === 'object') parent.appendChild(this.menu);
// @ts-ignore undefined
else document.getElementById(parent).appendChild(this.menu);
}
@ -133,11 +133,11 @@ class Menu {
}
get width() {
return this.menu.offsetWidth || 0;
return this.menu ? this.menu.offsetWidth : 0;
}
get height() {
return this.menu.offsetHeight || 0;
return this.menu ? this.menu.offsetHeight : 0;
}
hide() {
@ -184,6 +184,7 @@ class Menu {
this.hidden = !this.hidden;
const all = document.getElementsByClassName('menu');
for (const item of all) {
// @ts-ignore
item.style.display = this.hidden ? 'none' : 'block';
}
});
@ -205,8 +206,10 @@ class Menu {
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-checkbox-label" for="${this.ID}"></label></div>${title}`;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
if (evt.target) {
object[variable] = evt.target['checked'];
if (callback) callback(evt.target['checked']);
}
});
return el;
}
@ -225,7 +228,7 @@ class Menu {
el.style.fontVariant = document.body.style.fontVariant;
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (callback) callback(items[evt.target.selectedIndex]);
if (callback && evt.target) callback(items[evt.target['selectedIndex']]);
});
return el;
}
@ -237,12 +240,13 @@ class Menu {
if (this.container) this.container.appendChild(el);
el.addEventListener('change', (evt) => {
if (evt.target) {
object[variable] = parseInt(evt.target.value) === parseFloat(evt.target.value) ? parseInt(evt.target.value) : parseFloat(evt.target.value);
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target.value);
object[variable] = parseInt(evt.target['value']) === parseFloat(evt.target['value']) ? parseInt(evt.target['value']) : parseFloat(evt.target['value']);
// @ts-ignore
evt.target.setAttribute('value', evt.target['value']);
if (callback) callback(evt.target['value']);
}
});
el.input = el.children[0];
el['input'] = el.children[0];
return el;
}
@ -302,9 +306,12 @@ class Menu {
// eslint-disable-next-line class-methods-use-this
async updateChart(id, values) {
if (!values || (values.length === 0)) return;
/** @type {HTMLCanvasElement} */
// @ts-ignore undefined
const canvas = document.getElementById(`menu-canvas-${id}`);
if (!canvas) return;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.fillStyle = theme.background;
ctx.fillRect(0, 0, canvas.width, canvas.height);
const width = canvas.width / values.length;
@ -318,7 +325,7 @@ class Menu {
ctx.fillRect(i * width, 0, width - 4, canvas.height);
ctx.fillStyle = theme.background;
ctx.font = `${width / 1.5}px "Segoe UI"`;
ctx.fillText(Math.round(values[i]), i * width + 1, canvas.height - 1, width - 1);
ctx.fillText(Math.round(values[i]).toString(), i * width + 1, canvas.height - 1, width - 1);
}
}
}

13
demo/index-worker.js
View File

@ -1,12 +1,17 @@
/// <reference lib="webworker" />
/**
* Web worker used by main demo app
* Loaded from index.js
*/
/// <reference lib="webworker"/>
// load Human using IIFE script as Chome Mobile does not support Modules as Workers
// import Human from '../dist/human.esm.js';
/** @type {Human} */
const Human = {};
self.importScripts('../dist/human.js');
let busy = false;
// @ts-ignore // Human is registered as global namespace using IIFE script
// eslint-disable-next-line no-undef, new-cap
// eslint-disable-next-line new-cap
const human = new Human.default();
onmessage = async (msg) => { // receive message from main thread

56
demo/multithread/index.js
View File

@ -4,9 +4,8 @@
* @description Demo app that enables all Human modules and runs them in separate worker threads
*
*/
// @ts-nocheck // typescript checks disabled as this is pure javascript
import Human from '../../dist/human.esm.js'; // equivalent of @vladmandic/human
import Human from '../../dist/human.custom.esm.js'; // equivalent of @vladmandic/human
import GLBench from '../helpers/gl-bench.js';
const workerJS = './worker.js';
@ -92,9 +91,13 @@ const busy = {
};
const workers = {
/** @type {Worker | null} */
face: null,
/** @type {Worker | null} */
body: null,
/** @type {Worker | null} */
hand: null,
/** @type {Worker | null} */
object: null,
};
@ -138,7 +141,8 @@ async function drawResults() {
time.draw = Math.round(1 + human.now() - start.draw);
const fps = Math.round(10 * 1000 / time.main) / 10;
const draw = Math.round(10 * 1000 / time.draw) / 10;
document.getElementById('log').innerText = `Human: version ${human.version} | Performance: Main ${time.main}ms Face: ${time.face}ms Body: ${time.body}ms Hand: ${time.hand}ms Object ${time.object}ms | FPS: ${fps} / ${draw}`;
const div = document.getElementById('log');
if (div) div.innerText = `Human: version ${human.version} | Performance: Main ${time.main}ms Face: ${time.face}ms Body: ${time.body}ms Hand: ${time.hand}ms Object ${time.object}ms | FPS: ${fps} / ${draw}`;
requestAnimationFrame(drawResults);
}
@ -152,7 +156,7 @@ async function runDetection() {
start.main = human.now();
if (!bench) {
bench = new GLBench(null, { trackGPU: false, chartHz: 20, chartLen: 20 });
bench.begin();
bench.begin('human');
}
const ctx = canvas.getContext('2d');
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
@ -160,22 +164,22 @@ async function runDetection() {
if (!busy.face) {
busy.face = true;
start.face = human.now();
workers.face.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.face, type: 'face' }, [imageData.data.buffer.slice(0)]);
if (workers.face) workers.face.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.face, type: 'face' }, [imageData.data.buffer.slice(0)]);
}
if (!busy.body) {
busy.body = true;
start.body = human.now();
workers.body.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.body, type: 'body' }, [imageData.data.buffer.slice(0)]);
if (workers.body) workers.body.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.body, type: 'body' }, [imageData.data.buffer.slice(0)]);
}
if (!busy.hand) {
busy.hand = true;
start.hand = human.now();
workers.hand.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.hand, type: 'hand' }, [imageData.data.buffer.slice(0)]);
if (workers.hand) workers.hand.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.hand, type: 'hand' }, [imageData.data.buffer.slice(0)]);
}
if (!busy.object) {
busy.object = true;
start.object = human.now();
workers.object.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.object, type: 'object' }, [imageData.data.buffer.slice(0)]);
if (workers.object) workers.object.postMessage({ image: imageData.data.buffer, width: canvas.width, height: canvas.height, config: config.object, type: 'object' }, [imageData.data.buffer.slice(0)]);
}
time.main = Math.round(human.now() - start.main);
@ -204,27 +208,29 @@ async function setupCamera() {
try {
stream = await navigator.mediaDevices.getUserMedia(constraints);
} catch (err) {
output.innerText += `\n${err.name}: ${err.message}`;
status(err.name);
if (output) output.innerText += `\n${err.name}: ${err.message}`;
log('camera error:', err);
}
const tracks = stream.getVideoTracks();
log('enumerated viable tracks:', tracks);
const track = stream.getVideoTracks()[0];
const settings = track.getSettings();
log('selected video source:', track, settings);
if (stream) {
const tracks = stream.getVideoTracks();
log('enumerated viable tracks:', tracks);
const track = stream.getVideoTracks()[0];
const settings = track.getSettings();
log('selected video source:', track, settings);
} else {
log('missing video stream');
}
const promise = !stream || new Promise((resolve) => {
video.onloadeddata = () => {
if (settings.width > settings.height) canvas.style.width = '100vw';
else canvas.style.height = '100vh';
canvas.style.height = '100vh';
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
video.play();
resolve();
resolve(true);
};
});
// attach input to video element
if (stream) video.srcObject = stream;
if (stream && video) video['srcObject'] = stream;
return promise;
}
@ -240,21 +246,13 @@ async function startWorkers() {
}
async function main() {
window.addEventListener('unhandledrejection', (evt) => {
// eslint-disable-next-line no-console
console.error(evt.reason || evt);
document.getElementById('log').innerHTML = evt.reason.message || evt.reason || evt;
status('exception error');
evt.preventDefault();
});
if (typeof Worker === 'undefined' || typeof OffscreenCanvas === 'undefined') {
status('workers are not supported');
return;
}
human = new Human(config.main);
document.getElementById('log').innerText = `Human: version ${human.version}`;
const div = document.getElementById('log');
if (div) div.innerText = `Human: version ${human.version}`;
await startWorkers();
await setupCamera();

4
demo/multithread/worker.js
View File

@ -1,8 +1,6 @@
// load Human using IIFE script as Chome Mobile does not support Modules as Workers
/// <reference lib="webworker" />
// import Human from '../dist/human.esm.js';
// load Human using IIFE script as Chome Mobile does not support Modules as Workers
self.importScripts('../../dist/human.js');
let human;

5
package.json
View File

@ -66,16 +66,18 @@
"@tensorflow/tfjs-layers": "^3.10.0",
"@tensorflow/tfjs-node": "^3.10.0",
"@tensorflow/tfjs-node-gpu": "^3.10.0",
"@types/node": "^16.11.5",
"@types/node": "^16.11.6",
"@typescript-eslint/eslint-plugin": "^5.2.0",
"@typescript-eslint/parser": "^5.2.0",
"@vladmandic/build": "^0.6.3",
"@vladmandic/pilogger": "^0.3.3",
"canvas": "^2.8.0",
"dayjs": "^1.10.7",
"long": "^4.0.0",
"esbuild": "^0.13.9",
"eslint": "8.1.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-html": "^6.2.0",
"eslint-plugin-import": "^2.25.2",
"eslint-plugin-json": "^3.1.0",
"eslint-plugin-node": "^11.1.0",
@ -88,6 +90,5 @@
"typescript": "4.4.4"
},
"dependencies": {
"long": "^4.0.0"
}
}

17
src/face/match.ts
View File

@ -1,5 +1,6 @@
/** Face descriptor type as number array */
export type Descriptor = Array<number>
export type Options = { order?: number, threshold?: number, multiplier?: number } | undefined;
/** Calculates distance between two descriptors
* @param {object} options
@ -9,12 +10,12 @@ export type Descriptor = Array<number>
* - default is 20 which normalizes results to similarity above 0.5 can be considered a match
* @returns {number}
*/
export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) {
export function distance(descriptor1: Descriptor, descriptor2: Descriptor, options: Options = { order: 2, multiplier: 20 }) {
// general minkowski distance, euclidean distance is limited case where order is 2
let sum = 0;
for (let i = 0; i < descriptor1.length; i++) {
const diff = (options.order === 2) ? (descriptor1[i] - descriptor2[i]) : (Math.abs(descriptor1[i] - descriptor2[i]));
sum += (options.order === 2) ? (diff * diff) : (diff ** options.order);
const diff = (!options.order || options.order === 2) ? (descriptor1[i] - descriptor2[i]) : (Math.abs(descriptor1[i] - descriptor2[i]));
sum += (!options.order || options.order === 2) ? (diff * diff) : (diff ** options.order);
}
return (options.multiplier || 20) * sum;
}
@ -27,9 +28,9 @@ export function distance(descriptor1: Descriptor, descriptor2: Descriptor, optio
* - default is 20 which normalizes results to similarity above 0.5 can be considered a match
* @returns {number} similarity between two face descriptors normalized to 0..1 range where 0 is no similarity and 1 is perfect similarity
*/
export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options = { order: 2, multiplier: 20 }) {
export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, options: Options = { order: 2, multiplier: 20 }) {
const dist = distance(descriptor1, descriptor2, options);
const root = (options.order === 2) ? Math.sqrt(dist) : dist ** (1 / options.order);
const root = (!options.order || options.order === 2) ? Math.sqrt(dist) : dist ** (1 / options.order);
const invert = Math.max(0, 100 - root) / 100.0;
return invert;
}
@ -45,7 +46,7 @@ export function similarity(descriptor1: Descriptor, descriptor2: Descriptor, opt
* - {@link distance} calculated `distance` of given descriptor to the best match
* - {@link similarity} calculated normalized `similarity` of given descriptor to the best match
*/
export function match(descriptor: Descriptor, descriptors: Array<Descriptor>, options = { order: 2, threshold: 0, multiplier: 20 }) {
export function match(descriptor: Descriptor, descriptors: Array<Descriptor>, options: Options = { order: 2, multiplier: 20, threshold: 0 }) {
if (!Array.isArray(descriptor) || !Array.isArray(descriptors) || descriptor.length < 64 || descriptors.length === 0 || descriptor.length !== descriptors[0].length) { // validate input
return { index: -1, distance: Number.POSITIVE_INFINITY, similarity: 0 };
}
@ -57,8 +58,8 @@ export function match(descriptor: Descriptor, descriptors: Array<Descriptor>, op
best = res;
index = i;
}
if (best < options.threshold) break;
if (best < (options.threshold || 0)) break;
}
best = (options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order);
best = (!options.order || options.order === 2) ? Math.sqrt(best) : best ** (1 / options.order);
return { index, distance: best, similarity: Math.max(0, 100 - best) / 100.0 };
}

4
src/human.ts
View File

@ -212,7 +212,7 @@ export class Human {
}
/** Validate current configuration schema */
public validate(userConfig?: Partial<Config>) {
validate(userConfig?: Partial<Config>) {
return validate(defaults, userConfig || this.config);
}
@ -294,7 +294,7 @@ export class Human {
await tf.ready();
if (this.env.browser) {
if (this.config.debug) log('configuration:', this.config);
if (this.config.debug) log('tf flags:', this.tf.ENV.flags);
if (this.config.debug) log('tf flags:', this.tf.ENV['flags']);
}
}

15
test/browser.html
View File

@ -34,7 +34,7 @@
debug: true,
cacheSensitivity: 0,
object: { enabled: true },
}
};
const backends = ['wasm', 'webgl', 'humangl', 'webgpu'];
@ -55,7 +55,8 @@
const dt = new Date();
const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`;
const elap = (dt - last).toString().padStart(5, '0');
document.getElementById('log').innerHTML += ts + ' +' + elap + 'ms' + '&nbsp' + str(...msgs);
document.getElementById('log').innerHTML += ts + ' +' + elap + 'ms &nbsp' + str(...msgs);
// eslint-disable-next-line no-console
console.log(ts, elap, ...msgs);
last = dt;
}
@ -63,7 +64,7 @@
async function image(url) {
const el = document.createElement('img');
el.id = 'image';
const loaded = new Promise((resolve) => { el.onload = () => resolve(true) });
const loaded = new Promise((resolve) => { el.onload = () => resolve(true); });
el.src = url;
await loaded;
return el;
@ -88,7 +89,7 @@
async function main() {
log('human tests');
let res;
let human = new Human(config);
const human = new Human(config);
await human.init();
human.events.addEventListener('warmup', () => events('warmup'));
human.events.addEventListener('image', () => events('image'));
@ -110,7 +111,7 @@
log({ memory: human.tf.memory() });
res = await human.validate();
log({ validate: res });
res = await human.warmup({ warmup: 'face'});
res = await human.warmup({ warmup: 'face' });
draw(res.canvas);
log({ warmup: 'face' });
let img = await image('../../samples/in/ai-body.jpg');
@ -119,11 +120,11 @@
draw(res.canvas);
res = await human.detect(input.tensor);
log({ detect: true });
const interpolated = human.next();
human.next();
log({ interpolated: true });
const persons = res.persons;
log({ persons: true });
log({ summary: { persons: persons.length, face: res.face.length, body: res.body.length, hand: res.hand.length, object: res.object.length, gesture: res.gesture.length }});
log({ summary: { persons: persons.length, face: res.face.length, body: res.body.length, hand: res.hand.length, object: res.object.length, gesture: res.gesture.length } });
log({ performance: human.performance });
human.tf.dispose(input.tensor);
draw();

3
tsconfig.json
View File

@ -6,6 +6,7 @@
"outDir": "types",
"baseUrl": "./",
"paths": { "tslib": ["./node_modules/tslib/tslib.d.ts"] },
"lib": ["es2020", "dom", "webworker"],
"allowJs": true,
"allowSyntheticDefaultImports": false,
"allowUnreachableCode": false,
@ -50,7 +51,7 @@
"tabSize": 2
},
"exclude": ["node_modules/", "types/", "tfjs/", "dist/"],
"include": ["src"],
"include": ["src", "types/human.d.ts"],
"typedocOptions": {
"externalPattern": ["node_modules/", "tfjs/"]
}

2
wiki

@ -1 +1 @@
Subproject commit 82ade650a7cd593e29a98a8b8a1cba893e14c2f0
Subproject commit 20389b9779834324acbbcf2b25041a489a688d18