ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

package updates

pull/280/head
Vladimir Mandic 2021-10-12 14:17:33 -04:00
parent e7dce1311c
commit 569b1afcfd
11 changed files with 67 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -9,8 +9,12 @@
## Changelog ## Changelog
### **HEAD -> main** 2021/10/10 mandic00@live.com ### **HEAD -> main** 2021/10/12 mandic00@live.com
### **release: 2.3.2** 2021/10/11 mandic00@live.com
- major precision improvements to movenet and handtrack
- image processing fixes - image processing fixes
- redesign body and hand caching and interpolation - redesign body and hand caching and interpolation
- demo default config cleanup - demo default config cleanup

19
TODO.md
View File

@ -6,16 +6,13 @@
### Models ### Models
- Implement BlazePose end-to-end - Optimize BlazePose
- Add Anti-Spoofing
<br> <br>
### Backends ### Backends
#### WebGL
- Optimize shader packing for WebGL backend:
<https://github.com/tensorflow/tfjs/issues/5343>
#### WASM #### WASM
- Backend WASM incorrect handling of `int32` tensors - Backend WASM incorrect handling of `int32` tensors
@ -23,17 +20,9 @@
#### WebGPU #### WebGPU
Implementation of WebGPU backend
Experimental support only until support is officially added in Chromium Experimental support only until support is officially added in Chromium
- Performance issues:
- Evaluate WGSL vs GLSL for WebGPU <https://github.com/tensorflow/tfjs/issues/5689>
- Backend WebGPU missing kernel ops
<https://github.com/tensorflow/tfjs/issues/5496>
- Backend WebGPU incompatible with web workers
<https://github.com/tensorflow/tfjs/issues/5467>
- Backend WebGPU incompatible with sync read calls
<https://github.com/tensorflow/tfjs/issues/5468>
<br> <br>

10
demo/facematch/facematch.js
View File

@ -12,13 +12,13 @@ const userConfig = {
backend: 'wasm', backend: 'wasm',
async: false, async: false,
warmup: 'none', warmup: 'none',
cacheSimilarity: 0, cacheSensitivity: 0,
debug: true, debug: true,
modelBasePath: '../../models/', modelBasePath: '../../models/',
// wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/', // wasmPath: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@3.9.0/dist/',
face: { face: {
enabled: true, enabled: true,
detector: { rotation: true, return: true, maxDetected: 20 }, detector: { rotation: true, return: true, maxDetected: 50 },
mesh: { enabled: true }, mesh: { enabled: true },
embedding: { enabled: false }, embedding: { enabled: false },
iris: { enabled: true }, iris: { enabled: true },
@ -120,7 +120,7 @@ async function SelectFaceCanvas(face) {
ctx.font = 'small-caps 1rem "Lato"'; ctx.font = 'small-caps 1rem "Lato"';
const start = performance.now(); const start = performance.now();
const arr = db.map((rec) => rec.embedding); const arr = db.map((rec) => rec.embedding);
const res = await human.match(face.embedding, arr); const res = await human.match(current.embedding, arr);
time += (performance.now() - start); time += (performance.now() - start);
if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30); if (res.similarity > minScore) ctx.fillText(`DB: ${(100 * res.similarity).toFixed(1)}% ${db[res.index].name}`, 4, canvas.height - 30);
} }
@ -226,7 +226,7 @@ async function main() {
// could not dynamically enumerate images so using static list // could not dynamically enumerate images so using static list
if (images.length === 0) { if (images.length === 0) {
images = [ images = [
'ai-body.jpg', 'ai-upper.jpg', 'ai-body.jpg', 'solvay1927.jpg', 'ai-upper.jpg',
'person-carolina.jpg', 'person-celeste.jpg', 'person-leila1.jpg', 'person-leila2.jpg', 'person-lexi.jpg', 'person-linda.jpg', 'person-nicole.jpg', 'person-tasia.jpg', 'person-carolina.jpg', 'person-celeste.jpg', 'person-leila1.jpg', 'person-leila2.jpg', 'person-lexi.jpg', 'person-linda.jpg', 'person-nicole.jpg', 'person-tasia.jpg',
'person-tetiana.jpg', 'person-vlado1.jpg', 'person-vlado5.jpg', 'person-vlado.jpg', 'person-christina.jpg', 'person-lauren.jpg', 'person-tetiana.jpg', 'person-vlado1.jpg', 'person-vlado5.jpg', 'person-vlado.jpg', 'person-christina.jpg', 'person-lauren.jpg',
'group-1.jpg', 'group-2.jpg', 'group-3.jpg', 'group-4.jpg', 'group-5.jpg', 'group-6.jpg', 'group-7.jpg', 'group-1.jpg', 'group-2.jpg', 'group-3.jpg', 'group-4.jpg', 'group-5.jpg', 'group-6.jpg', 'group-7.jpg',
@ -243,6 +243,8 @@ async function main() {
log('Discovered images:', images); log('Discovered images:', images);
} }
// images = ['/samples/in/solvay1927.jpg'];
// download and analyze all images // download and analyze all images
for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length); for (let i = 0; i < images.length; i++) await AddImageElement(i, images[i], images.length);

26
demo/facematch/faces.json
View File

File diff suppressed because one or more lines are too long

3
demo/facematch/index.html
View File

@ -1,8 +1,9 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="utf-8">
<title>Human</title> <title>Human</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8"> <!-- <meta http-equiv="content-type" content="text/html; charset=utf-8"> -->
<meta name="viewport" content="width=device-width, shrink-to-fit=yes"> <meta name="viewport" content="width=device-width, shrink-to-fit=yes">
<meta name="keywords" content="Human"> <meta name="keywords" content="Human">
<meta name="application-name" content="Human"> <meta name="application-name" content="Human">

2
demo/index.js
View File

@ -146,7 +146,7 @@ let bench;
let lastDetectedResult = {}; let lastDetectedResult = {};
// helper function: async pause // helper function: async pause
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
// helper function: translates json to human readable string // helper function: translates json to human readable string

2
demo/manifest.webmanifest
View File

@ -7,4 +7,4 @@
"display": "standalone", "display": "standalone",
"background_color": "#000000", "background_color": "#000000",
"theme_color": "#000000" "theme_color": "#000000"
} }

18
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@vladmandic/human", "name": "@vladmandic/human",
"version": "2.3.2", "version": "2.3.3",
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
"sideEffects": false, "sideEffects": false,
"main": "dist/human.node.js", "main": "dist/human.node.js",
@ -55,7 +55,6 @@
"tensorflow" "tensorflow"
], ],
"devDependencies": { "devDependencies": {
"@tensorflow/tfjs": "^3.9.0",
"@tensorflow/tfjs-backend-cpu": "^3.9.0", "@tensorflow/tfjs-backend-cpu": "^3.9.0",
"@tensorflow/tfjs-backend-wasm": "^3.9.0", "@tensorflow/tfjs-backend-wasm": "^3.9.0",
"@tensorflow/tfjs-backend-webgl": "^3.9.0", "@tensorflow/tfjs-backend-webgl": "^3.9.0",
@ -64,21 +63,23 @@
"@tensorflow/tfjs-core": "^3.9.0", "@tensorflow/tfjs-core": "^3.9.0",
"@tensorflow/tfjs-data": "^3.9.0", "@tensorflow/tfjs-data": "^3.9.0",
"@tensorflow/tfjs-layers": "^3.9.0", "@tensorflow/tfjs-layers": "^3.9.0",
"@tensorflow/tfjs-node": "^3.9.0",
"@tensorflow/tfjs-node-gpu": "^3.9.0", "@tensorflow/tfjs-node-gpu": "^3.9.0",
"@types/node": "^16.10.3", "@tensorflow/tfjs-node": "^3.9.0",
"@typescript-eslint/eslint-plugin": "^4.33.0", "@tensorflow/tfjs": "^3.9.0",
"@typescript-eslint/parser": "^4.33.0", "@types/node": "^16.10.4",
"@vladmandic/build": "^0.5.3", "@typescript-eslint/eslint-plugin": "^5.0.0",
"@typescript-eslint/parser": "^5.0.0",
"@vladmandic/build": "^0.6.0",
"@vladmandic/pilogger": "^0.3.3", "@vladmandic/pilogger": "^0.3.3",
"canvas": "^2.8.0", "canvas": "^2.8.0",
"dayjs": "^1.10.7", "dayjs": "^1.10.7",
"esbuild": "^0.13.4", "esbuild": "^0.13.4",
"eslint-config-airbnb-base": "^14.2.1", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.24.2", "eslint-plugin-import": "^2.25.1",
"eslint-plugin-json": "^3.1.0", "eslint-plugin-json": "^3.1.0",
"eslint-plugin-node": "^11.1.0", "eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^5.1.0", "eslint-plugin-promise": "^5.1.0",
"eslint": "8.0.0",
"node-fetch": "^3.0.0", "node-fetch": "^3.0.0",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"seedrandom": "^3.0.5", "seedrandom": "^3.0.5",
@ -87,6 +88,5 @@
"typescript": "4.4.3" "typescript": "4.4.3"
}, },
"dependencies": { "dependencies": {
"eslint": "7.32.0"
} }
} }

BIN
samples/in/solvay1927.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 211 KiB

6
src/human.ts
View File

@ -227,7 +227,7 @@ export class Human {
this.#numTensors = currentTensors; this.#numTensors = currentTensors;
const leaked = currentTensors - previousTensors; const leaked = currentTensors - previousTensors;
if (leaked !== 0) log(...msg, leaked); if (leaked !== 0) log(...msg, leaked);
} };
// quick sanity check on inputs // quick sanity check on inputs
/** @hidden */ /** @hidden */
@ -241,7 +241,7 @@ export class Human {
return 'backend not loaded'; return 'backend not loaded';
} }
return null; return null;
} };
/** Reset configuration to default values */ /** Reset configuration to default values */
reset(): void { reset(): void {
@ -350,7 +350,7 @@ export class Human {
/** @hidden */ /** @hidden */
emit = (event: string) => { emit = (event: string) => {
if (this.events && this.events.dispatchEvent) this.events?.dispatchEvent(new Event(event)); if (this.events && this.events.dispatchEvent) this.events?.dispatchEvent(new Event(event));
} };
/** Runs interpolation using last known result and returns smoothened result /** Runs interpolation using last known result and returns smoothened result
* Interpolation is based on time since last known result so can be called independently * Interpolation is based on time since last known result so can be called independently

22
src/image/imagefx.ts
View File

@ -5,6 +5,14 @@
import * as shaders from './imagefxshaders'; import * as shaders from './imagefxshaders';
const collect = (source, prefix, collection) => {
const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig');
source.replace(r, (match, name) => {
collection[name] = 0;
return match;
});
};
class GLProgram { class GLProgram {
uniform = {}; uniform = {};
attribute = {}; attribute = {};
@ -20,21 +28,13 @@ class GLProgram {
this.gl.linkProgram(this.id); this.gl.linkProgram(this.id);
if (!this.gl.getProgramParameter(this.id, this.gl.LINK_STATUS)) throw new Error(`filter: gl link failed: ${this.gl.getProgramInfoLog(this.id)}`); if (!this.gl.getProgramParameter(this.id, this.gl.LINK_STATUS)) throw new Error(`filter: gl link failed: ${this.gl.getProgramInfoLog(this.id)}`);
this.gl.useProgram(this.id); this.gl.useProgram(this.id);
this.collect(vertexSource, 'attribute', this.attribute); // Collect attributes collect(vertexSource, 'attribute', this.attribute); // Collect attributes
for (const a in this.attribute) this.attribute[a] = this.gl.getAttribLocation(this.id, a); for (const a in this.attribute) this.attribute[a] = this.gl.getAttribLocation(this.id, a);
this.collect(vertexSource, 'uniform', this.uniform); // Collect uniforms collect(vertexSource, 'uniform', this.uniform); // Collect uniforms
this.collect(fragmentSource, 'uniform', this.uniform); collect(fragmentSource, 'uniform', this.uniform);
for (const u in this.uniform) this.uniform[u] = this.gl.getUniformLocation(this.id, u); for (const u in this.uniform) this.uniform[u] = this.gl.getUniformLocation(this.id, u);
} }
collect = (source, prefix, collection) => {
const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig');
source.replace(r, (match, name) => {
collection[name] = 0;
return match;
});
};
compile = (source, type): WebGLShader => { compile = (source, type): WebGLShader => {
const shader = this.gl.createShader(type) as WebGLShader; const shader = this.gl.createShader(type) as WebGLShader;
this.gl.shaderSource(shader, source); this.gl.shaderSource(shader, source);