ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update imagefx

pull/70/head
Vladimir Mandic 2021-02-19 08:35:41 -05:00
parent 9d4a9b66a0
commit cd16dcce49
25 changed files with 1380 additions and 1438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@ -1,6 +1,16 @@
# Human Library # Human Library
## 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition ### 3D Face Detection, Face Embedding & Recognition,
### Body Pose Tracking, Hand & Finger Tracking,
### Iris Analysis, Age & Gender & Emotion Prediction
### & Gesture Recognition
<br>
Native JavaScript module using TensorFlow/JS Machine Learning library
Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows and Linux
- Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends
- NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu*
<br> <br>
@ -39,28 +49,17 @@
<br> <br>
Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows and Linux
- Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends
- NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu*
- (and maybe with React-Native as it doesn't use any DOM objects)
<br>
*See [issues](https://github.com/vladmandic/human/issues?q=) and [discussions](https://github.com/vladmandic/human/discussions) for list of known limitations and planned enhancements* *See [issues](https://github.com/vladmandic/human/issues?q=) and [discussions](https://github.com/vladmandic/human/discussions) for list of known limitations and planned enhancements*
*Suggestions are welcome!* *Suggestions are welcome!*
<br> <br><hr><br>
<hr>
<br>
## Options ## ## Options ##
![Options visible in demo](assets/screenshot-menu.png) ![Options visible in demo](assets/screenshot-menu.png)
<br> <br><hr><br>
<hr>
<br>
## Examples ## Examples
@ -75,4 +74,3 @@ Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows an
**Using webcam:** **Using webcam:**
![Example Using WebCam](assets/screenshot2.jpg) ![Example Using WebCam](assets/screenshot2.jpg)

7
demo/browser.js
View File

@ -1,4 +1,9 @@
import Human from '../dist/human.esm.js'; // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
// import * as tf from '@tensorflow/tfjs';
// import Human from '../dist/human.esm-nobundle.js';
import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
import draw from './draw.js'; import draw from './draw.js';
import Menu from './menu.js'; import Menu from './menu.js';
import GLBench from './gl-bench.js'; import GLBench from './gl-bench.js';

520
dist/demo-browser-index.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/demo-browser-index.js.map vendored
View File

File diff suppressed because one or more lines are too long

10
dist/demo-browser-index.json vendored
View File

@ -1,7 +1,7 @@
{ {
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytes": 1347254, "bytes": 1347469,
"imports": [] "imports": []
}, },
"demo/draw.js": { "demo/draw.js": {
@ -17,7 +17,7 @@
"imports": [] "imports": []
}, },
"demo/browser.js": { "demo/browser.js": {
"bytes": 26540, "bytes": 26752,
"imports": [ "imports": [
{ {
"path": "dist/human.esm.js", "path": "dist/human.esm.js",
@ -43,14 +43,14 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 2021929 "bytes": 2022314
}, },
"dist/demo-browser-index.js": { "dist/demo-browser-index.js": {
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": { "inputs": {
"dist/human.esm.js": { "dist/human.esm.js": {
"bytesInOutput": 1339760 "bytesInOutput": 1339993
}, },
"demo/draw.js": { "demo/draw.js": {
"bytesInOutput": 6204 "bytesInOutput": 6204
@ -65,7 +65,7 @@
"bytesInOutput": 17340 "bytesInOutput": 17340
} }
}, },
"bytes": 1388144 "bytes": 1388377
} }
} }
} }

20
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

892
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

28
dist/human.esm.json vendored
View File

@ -5,7 +5,7 @@
"imports": [] "imports": []
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1073705, "bytes": 1073613,
"imports": [] "imports": []
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
@ -388,11 +388,11 @@
"imports": [] "imports": []
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19447, "bytes": 19372,
"imports": [] "imports": []
}, },
"src/image.ts": { "src/image.ts": {
"bytes": 5867, "bytes": 5902,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -417,11 +417,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2529, "bytes": 2560,
"imports": [] "imports": []
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 19583, "bytes": 19703,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -499,7 +499,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1924959 "bytes": 1925031
}, },
"dist/human.esm.js": { "dist/human.esm.js": {
"imports": [], "imports": [],
@ -513,14 +513,11 @@
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1690 "bytesInOutput": 1690
}, },
"src/imagefx.js": {
"bytesInOutput": 11016
},
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1064700 "bytesInOutput": 1064608
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1205 "bytesInOutput": 1205
@ -541,7 +538,7 @@
"bytesInOutput": 5040 "bytesInOutput": 5040
}, },
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 10311 "bytesInOutput": 10617
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1549 "bytesInOutput": 1549
@ -609,8 +606,11 @@
"src/gesture/gesture.ts": { "src/gesture/gesture.ts": {
"bytesInOutput": 2391 "bytesInOutput": 2391
}, },
"src/imagefx.js": {
"bytesInOutput": 10975
},
"src/image.ts": { "src/image.ts": {
"bytesInOutput": 2454 "bytesInOutput": 2452
}, },
"config.js": { "config.js": {
"bytesInOutput": 1426 "bytesInOutput": 1426
@ -619,10 +619,10 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2528 "bytesInOutput": 2572
} }
}, },
"bytes": 1347254 "bytes": 1347469
} }
} }
} }

20
dist/human.node-gpu.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node-gpu.js.map vendored
View File

File diff suppressed because one or more lines are too long

20
dist/human.node.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.node.js.map vendored
View File

File diff suppressed because one or more lines are too long

28
dist/human.node.json vendored
View File

@ -388,11 +388,11 @@
"imports": [] "imports": []
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19447, "bytes": 19372,
"imports": [] "imports": []
}, },
"src/image.ts": { "src/image.ts": {
"bytes": 5867, "bytes": 5902,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -417,11 +417,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2529, "bytes": 2560,
"imports": [] "imports": []
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 19583, "bytes": 19703,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -499,7 +499,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 705489 "bytes": 705616
}, },
"dist/human.node-gpu.js": { "dist/human.node-gpu.js": {
"imports": [], "imports": [],
@ -514,11 +514,8 @@
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1677 "bytesInOutput": 1677
}, },
"src/imagefx.js": {
"bytesInOutput": 11012
},
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 10478 "bytesInOutput": 10625
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 251 "bytesInOutput": 251
@ -542,7 +539,7 @@
"bytesInOutput": 28973 "bytesInOutput": 28973
}, },
"src/faceboxes/faceboxes.ts": { "src/faceboxes/faceboxes.ts": {
"bytesInOutput": 1591 "bytesInOutput": 1586
}, },
"src/profile.ts": { "src/profile.ts": {
"bytesInOutput": 604 "bytesInOutput": 604
@ -551,7 +548,7 @@
"bytesInOutput": 822 "bytesInOutput": 822
}, },
"src/gender/gender.ts": { "src/gender/gender.ts": {
"bytesInOutput": 1310 "bytesInOutput": 1302
}, },
"src/emotion/emotion.ts": { "src/emotion/emotion.ts": {
"bytesInOutput": 1247 "bytesInOutput": 1247
@ -608,7 +605,10 @@
"bytesInOutput": 2391 "bytesInOutput": 2391
}, },
"src/image.ts": { "src/image.ts": {
"bytesInOutput": 2455 "bytesInOutput": 2458
},
"src/imagefx.js": {
"bytesInOutput": 10973
}, },
"config.js": { "config.js": {
"bytesInOutput": 1426 "bytesInOutput": 1426
@ -617,10 +617,10 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2525 "bytesInOutput": 2569
} }
}, },
"bytes": 276897 "bytes": 277039
} }
} }
} }

892
dist/human.ts vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.ts.map vendored
View File

File diff suppressed because one or more lines are too long

28
dist/human.tson vendored
View File

@ -5,7 +5,7 @@
"imports": [] "imports": []
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytes": 1073705, "bytes": 1073613,
"imports": [] "imports": []
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
@ -388,11 +388,11 @@
"imports": [] "imports": []
}, },
"src/imagefx.js": { "src/imagefx.js": {
"bytes": 19447, "bytes": 19372,
"imports": [] "imports": []
}, },
"src/image.ts": { "src/image.ts": {
"bytes": 5867, "bytes": 5902,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -417,11 +417,11 @@
"imports": [] "imports": []
}, },
"package.json": { "package.json": {
"bytes": 2529, "bytes": 2560,
"imports": [] "imports": []
}, },
"src/human.ts": { "src/human.ts": {
"bytes": 19583, "bytes": 19703,
"imports": [ "imports": [
{ {
"path": "src/log.ts", "path": "src/log.ts",
@ -499,7 +499,7 @@
"imports": [], "imports": [],
"exports": [], "exports": [],
"inputs": {}, "inputs": {},
"bytes": 1924970 "bytes": 1925042
}, },
"dist/human.ts": { "dist/human.ts": {
"imports": [], "imports": [],
@ -511,17 +511,14 @@
"src/posenet/keypoints.ts": { "src/posenet/keypoints.ts": {
"bytesInOutput": 1690 "bytesInOutput": 1690
}, },
"src/imagefx.js": {
"bytesInOutput": 11016
},
"src/human.ts": { "src/human.ts": {
"bytesInOutput": 10347 "bytesInOutput": 10653
}, },
"src/log.ts": { "src/log.ts": {
"bytesInOutput": 252 "bytesInOutput": 252
}, },
"dist/tfjs.esm.js": { "dist/tfjs.esm.js": {
"bytesInOutput": 1064700 "bytesInOutput": 1064608
}, },
"src/tfjs/backend.ts": { "src/tfjs/backend.ts": {
"bytesInOutput": 1205 "bytesInOutput": 1205
@ -607,8 +604,11 @@
"src/gesture/gesture.ts": { "src/gesture/gesture.ts": {
"bytesInOutput": 2391 "bytesInOutput": 2391
}, },
"src/imagefx.js": {
"bytesInOutput": 10975
},
"src/image.ts": { "src/image.ts": {
"bytesInOutput": 2454 "bytesInOutput": 2452
}, },
"config.js": { "config.js": {
"bytesInOutput": 1426 "bytesInOutput": 1426
@ -617,10 +617,10 @@
"bytesInOutput": 55295 "bytesInOutput": 55295
}, },
"package.json": { "package.json": {
"bytesInOutput": 2528 "bytesInOutput": 2572
} }
}, },
"bytes": 1347296 "bytes": 1347511
} }
} }
} }

24
package-lock.json generated
View File

@ -366,9 +366,9 @@
"dev": true "dev": true
}, },
"@types/node": { "@types/node": {
"version": "14.14.28", "version": "14.14.30",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.28.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.30.tgz",
"integrity": "sha512-lg55ArB+ZiHHbBBttLpzD07akz0QPrZgUODNakeC09i62dnrywr9mFErHuaPlB6I7z+sEbK+IYmplahvplCj2g==", "integrity": "sha512-gUWhy8s45fQp4PqqKecsnOkdW0kt1IaKjgOIR3HPokkzTmQj9ji2wWFID5THu1MKrtO+d4s2lVrlEhXUsPXSvg==",
"dev": true "dev": true
}, },
"@types/node-fetch": { "@types/node-fetch": {
@ -1026,9 +1026,9 @@
} }
}, },
"esbuild": { "esbuild": {
"version": "0.8.46", "version": "0.8.49",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.8.46.tgz", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.8.49.tgz",
"integrity": "sha512-xck9sXNCNmjDHCCfxTCyhKTiFuEBweh+IDAhMLOJI990v1Fzii6MyIkT1LbkvjgoVgPX2SK1kpi5eZVGNrl8yg==", "integrity": "sha512-itiFVYv5UZz4NooO7/Y0bRGVDGz/M/cxKbl6zyNI5pnKaz1mZjvZXAFhhDVz6rGCmcdTKj5oag6rh8DaaSSmfQ==",
"dev": true "dev": true
}, },
"escalade": { "escalade": {
@ -2572,9 +2572,9 @@
"dev": true "dev": true
}, },
"simple-git": { "simple-git": {
"version": "2.35.0", "version": "2.35.1",
"resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.35.0.tgz", "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.35.1.tgz",
"integrity": "sha512-VuXs2/HyZmZm43Z5IjvU+ahTmURh/Hmb/egmgNdFZuu8OEnW2emCalnL/4jRQkXeJvfzCTnev6wo5jtDmWw0Dw==", "integrity": "sha512-Y5/hXf5ivfMziWRNGhVsbiG+1h4CkTW2qVC3dRidLuSZYAPFbLCPP1d7rgiL40lgRPhPTBuhVzNJAV9glWstEg==",
"dev": true, "dev": true,
"requires": { "requires": {
"@kwsites/file-exists": "^1.1.1", "@kwsites/file-exists": "^1.1.1",
@ -2837,9 +2837,9 @@
"dev": true "dev": true
}, },
"typescript": { "typescript": {
"version": "4.3.0-dev.20210217", "version": "4.3.0-dev.20210219",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.0-dev.20210217.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.0-dev.20210219.tgz",
"integrity": "sha512-gjh5m/gbj9hVmoTzb3yePdhCfw96aAXhiV4MqyYMyvlZzaw+IkIOGvQBc8Q9Vu72jdVE3mClIzrIg3o/Xw/vVw==", "integrity": "sha512-9ttM1StIx+z2L4NnLGaYkOKIJWWFwzL4KuvhQXfulqjl4PnNjAsrl5QKAbI07oEvS2wn4rUdGHYQnCxh3/biWQ==",
"dev": true "dev": true
}, },
"uri-js": { "uri-js": {

9
package.json
View File

@ -6,6 +6,7 @@
"main": "dist/human.node.js", "main": "dist/human.node.js",
"module": "dist/human.esm.js", "module": "dist/human.esm.js",
"browser": "dist/human.esm.js", "browser": "dist/human.esm.js",
"types": "types/human.d.ts",
"author": "Vladimir Mandic <mandic00@live.com>", "author": "Vladimir Mandic <mandic00@live.com>",
"bugs": { "bugs": {
"url": "https://github.com/vladmandic/human/issues" "url": "https://github.com/vladmandic/human/issues"
@ -32,13 +33,13 @@
"@tensorflow/tfjs-layers": "^3.1.0", "@tensorflow/tfjs-layers": "^3.1.0",
"@tensorflow/tfjs-node": "^3.1.0", "@tensorflow/tfjs-node": "^3.1.0",
"@tensorflow/tfjs-node-gpu": "^3.1.0", "@tensorflow/tfjs-node-gpu": "^3.1.0",
"@types/node": "^14.14.28", "@types/node": "^14.14.30",
"@typescript-eslint/eslint-plugin": "^4.15.1", "@typescript-eslint/eslint-plugin": "^4.15.1",
"@typescript-eslint/parser": "^4.15.1", "@typescript-eslint/parser": "^4.15.1",
"@vladmandic/pilogger": "^0.2.14", "@vladmandic/pilogger": "^0.2.14",
"chokidar": "^3.5.1", "chokidar": "^3.5.1",
"dayjs": "^1.10.4", "dayjs": "^1.10.4",
"esbuild": "^0.8.46", "esbuild": "^0.8.49",
"eslint": "^7.20.0", "eslint": "^7.20.0",
"eslint-config-airbnb-base": "^14.2.1", "eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.22.1", "eslint-plugin-import": "^2.22.1",
@ -47,9 +48,9 @@
"eslint-plugin-promise": "^4.3.1", "eslint-plugin-promise": "^4.3.1",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"seedrandom": "^3.0.5", "seedrandom": "^3.0.5",
"simple-git": "^2.35.0", "simple-git": "^2.35.1",
"tslib": "^2.1.0", "tslib": "^2.1.0",
"typescript": "^4.3.0-dev.20210217" "typescript": "^4.3.0-dev.20210219"
}, },
"scripts": { "scripts": {
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js", "start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",

2
server/build.js
View File

@ -209,7 +209,7 @@ async function build(f, msg) {
for (const [targetGroupName, targetGroup] of Object.entries(targets)) { for (const [targetGroupName, targetGroup] of Object.entries(targets)) {
for (const [targetName, targetOptions] of Object.entries(targetGroup)) { for (const [targetName, targetOptions] of Object.entries(targetGroup)) {
// if triggered from watch mode, rebuild only browser bundle // if triggered from watch mode, rebuild only browser bundle
if ((require.main !== module) && (targetGroupName !== 'browserBundle')) continue; // if ((require.main !== module) && (targetGroupName !== 'browserBundle')) continue;
await es.build({ ...common, ...targetOptions }); await es.build({ ...common, ...targetOptions });
const stats = await getStats(targetOptions.metafile); const stats = await getStats(targetOptions.metafile);
log.state(`Build for: ${targetGroupName} type: ${targetName}:`, stats); log.state(`Build for: ${targetGroupName} type: ${targetName}:`, stats);

44
src/human.ts
View File

@ -101,7 +101,7 @@ class Human {
// helper function: measure tensor leak // helper function: measure tensor leak
analyze(...msg) { analyze(...msg) {
if (!this.analyzeMemoryLeaks) return; if (!this.analyzeMemoryLeaks) return;
const current = tf.engine().state.numTensors; const current = this.tf.engine().state.numTensors;
const previous = this.numTensors; const previous = this.numTensors;
this.numTensors = current; this.numTensors = current;
const leaked = current - previous; const leaked = current - previous;
@ -112,11 +112,11 @@ class Human {
sanity(input) { sanity(input) {
if (!this.checkSanity) return null; if (!this.checkSanity) return null;
if (!input) return 'input is not defined'; if (!input) return 'input is not defined';
if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) { if (this.tf.ENV.flags.IS_NODE && !(input instanceof this.tf.Tensor)) {
return 'input must be a tensor'; return 'input must be a tensor';
} }
try { try {
tf.getBackend(); this.tf.getBackend();
} catch { } catch {
return 'backend not loaded'; return 'backend not loaded';
} }
@ -135,11 +135,11 @@ class Human {
if (userConfig) this.config = mergeDeep(this.config, userConfig); if (userConfig) this.config = mergeDeep(this.config, userConfig);
if (this.firstRun) { if (this.firstRun) {
log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`); log(`version: ${this.version} TensorFlow/JS version: ${this.tf.version_core}`);
await this.checkBackend(true); await this.checkBackend(true);
if (tf.ENV.flags.IS_BROWSER) { if (this.tf.ENV.flags.IS_BROWSER) {
log('configuration:', this.config); log('configuration:', this.config);
log('tf flags:', tf.ENV.flags); log('tf flags:', this.tf.ENV.flags);
} }
} }
const face = this.config.face.detector.modelPath.includes('faceboxes') ? faceboxes : facemesh; const face = this.config.face.detector.modelPath.includes('faceboxes') ? faceboxes : facemesh;
@ -172,7 +172,7 @@ class Human {
} }
if (this.firstRun) { if (this.firstRun) {
log('tf engine state:', tf.engine().state.numBytes, 'bytes', tf.engine().state.numTensors, 'tensors'); log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors');
this.firstRun = false; this.firstRun = false;
} }
@ -182,7 +182,7 @@ class Human {
// check if backend needs initialization if it changed // check if backend needs initialization if it changed
async checkBackend(force = false) { async checkBackend(force = false) {
if (this.config.backend && (this.config.backend !== '') && force || (tf.getBackend() !== this.config.backend)) { if (this.config.backend && (this.config.backend !== '') && force || (this.tf.getBackend() !== this.config.backend)) {
const timeStamp = now(); const timeStamp = now();
this.state = 'backend'; this.state = 'backend';
/* force backend reload /* force backend reload
@ -199,32 +199,32 @@ class Human {
if (this.config.backend === 'wasm') { if (this.config.backend === 'wasm') {
log('settings wasm path:', this.config.wasmPath); log('settings wasm path:', this.config.wasmPath);
tf.setWasmPaths(this.config.wasmPath); this.tf.setWasmPaths(this.config.wasmPath);
const simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT'); const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
if (!simd) log('warning: wasm simd support is not enabled'); if (!simd) log('warning: wasm simd support is not enabled');
} }
if (this.config.backend === 'humangl') backend.register(); if (this.config.backend === 'humangl') backend.register();
try { try {
await tf.setBackend(this.config.backend); await this.tf.setBackend(this.config.backend);
} catch (err) { } catch (err) {
log('error: cannot set backend:', this.config.backend, err); log('error: cannot set backend:', this.config.backend, err);
} }
tf.enableProdMode(); this.tf.enableProdMode();
/* debug mode is really too mcuh /* debug mode is really too mcuh
tf.enableDebugMode(); tf.enableDebugMode();
*/ */
if (tf.getBackend() === 'webgl') { if (this.tf.getBackend() === 'webgl') {
if (this.config.deallocate) { if (this.config.deallocate) {
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate); log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1); this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
} }
tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); this.tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true); this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
const gl = await tf.backend().getGPGPUContext().gl; const gl = await this.tf.backend().getGPGPUContext().gl;
log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`); log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`);
} }
await tf.ready(); await this.tf.ready();
this.perf.backend = Math.trunc(now() - timeStamp); this.perf.backend = Math.trunc(now() - timeStamp);
} }
} }
@ -384,7 +384,7 @@ class Human {
// load models if enabled // load models if enabled
await this.load(); await this.load();
if (this.config.scoped) tf.engine().startScope(); if (this.config.scoped) this.tf.engine().startScope();
this.analyze('Start Scope:'); this.analyze('Start Scope:');
timeStamp = now(); timeStamp = now();
@ -440,7 +440,7 @@ class Human {
} }
process.tensor.dispose(); process.tensor.dispose();
if (this.config.scoped) tf.engine().endScope(); if (this.config.scoped) this.tf.engine().endScope();
this.analyze('End Scope:'); this.analyze('End Scope:');
let gestureRes = []; let gestureRes = [];
@ -512,10 +512,10 @@ class Human {
// @ts-ignore // @ts-ignore
const data = tf.node.decodeJpeg(img); // tf.node is only defined when compiling for nodejs const data = tf.node.decodeJpeg(img); // tf.node is only defined when compiling for nodejs
const expanded = data.expandDims(0); const expanded = data.expandDims(0);
tf.dispose(data); this.tf.dispose(data);
// log('Input:', expanded); // log('Input:', expanded);
const res = await this.detect(expanded, this.config); const res = await this.detect(expanded, this.config);
tf.dispose(expanded); this.tf.dispose(expanded);
return res; return res;
} }

1
src/image.ts
View File

@ -83,6 +83,7 @@ export function process(input, config) {
*/ */
} else { } else {
outCanvas = inCanvas; outCanvas = inCanvas;
if (this.fx) this.fx = null;
} }
let pixels; let pixels;
if (outCanvas.data) { if (outCanvas.data) {

190
src/imagefx.js
View File

@ -1,11 +1,10 @@
/* eslint-disable no-use-before-define */
/* /*
WebGLImageFilter - MIT Licensed WebGLImageFilter - MIT Licensed
2013, Dominic Szablewski - phoboslab.org 2013, Dominic Szablewski - phoboslab.org
<https://github.com/phoboslab/WebGLImageFilter> <https://github.com/phoboslab/WebGLImageFilter>
*/ */
const GLProgram = function (gl, vertexSource, fragmentSource) { function GLProgram(gl, vertexSource, fragmentSource) {
const _collect = function (source, prefix, collection) { const _collect = function (source, prefix, collection) {
const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig'); const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig');
source.replace(r, (match, name) => { source.replace(r, (match, name) => {
@ -18,7 +17,6 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
const shader = gl.createShader(type); const shader = gl.createShader(type);
gl.shaderSource(shader, source); gl.shaderSource(shader, source);
gl.compileShader(shader); gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
// @ts-ignore // @ts-ignore
throw new Error('Filter: GL compile failed', gl.getShaderInfoLog(shader)); throw new Error('Filter: GL compile failed', gl.getShaderInfoLog(shader));
@ -28,10 +26,8 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
this.uniform = {}; this.uniform = {};
this.attribute = {}; this.attribute = {};
const _vsh = _compile(vertexSource, gl.VERTEX_SHADER); const _vsh = _compile(vertexSource, gl.VERTEX_SHADER);
const _fsh = _compile(fragmentSource, gl.FRAGMENT_SHADER); const _fsh = _compile(fragmentSource, gl.FRAGMENT_SHADER);
this.id = gl.createProgram(); this.id = gl.createProgram();
gl.attachShader(this.id, _vsh); gl.attachShader(this.id, _vsh);
gl.attachShader(this.id, _fsh); gl.attachShader(this.id, _fsh);
@ -43,22 +39,17 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
} }
gl.useProgram(this.id); gl.useProgram(this.id);
// Collect attributes // Collect attributes
_collect(vertexSource, 'attribute', this.attribute); _collect(vertexSource, 'attribute', this.attribute);
for (const a in this.attribute) { for (const a in this.attribute) this.attribute[a] = gl.getAttribLocation(this.id, a);
this.attribute[a] = gl.getAttribLocation(this.id, a);
}
// Collect uniforms // Collect uniforms
_collect(vertexSource, 'uniform', this.uniform); _collect(vertexSource, 'uniform', this.uniform);
_collect(fragmentSource, 'uniform', this.uniform); _collect(fragmentSource, 'uniform', this.uniform);
for (const u in this.uniform) { for (const u in this.uniform) this.uniform[u] = gl.getUniformLocation(this.id, u);
this.uniform[u] = gl.getUniformLocation(this.id, u); }
}
};
const GLImageFilter = function (params) { // export const GLImageFilter = function (params) {
export function GLImageFilter(params) {
if (!params) params = { }; if (!params) params = { };
let _drawCount = 0; let _drawCount = 0;
let _sourceTexture = null; let _sourceTexture = null;
@ -70,11 +61,11 @@ const GLImageFilter = function (params) {
let _height = -1; let _height = -1;
let _vertexBuffer = null; let _vertexBuffer = null;
let _currentProgram = null; let _currentProgram = null;
const _filter = {};
const _canvas = params.canvas || document.createElement('canvas'); const _canvas = params.canvas || document.createElement('canvas');
// key is the shader program source, value is the compiled program // key is the shader program source, value is the compiled program
const _shaderProgramCache = { }; const _shaderProgramCache = { };
const DRAW = { INTERMEDIATE: 1 };
const gl = _canvas.getContext('webgl'); const gl = _canvas.getContext('webgl');
if (!gl) throw new Error('Filter: getContext() failed'); if (!gl) throw new Error('Filter: getContext() failed');
@ -82,7 +73,6 @@ const GLImageFilter = function (params) {
// eslint-disable-next-line prefer-rest-params // eslint-disable-next-line prefer-rest-params
const args = Array.prototype.slice.call(arguments, 1); const args = Array.prototype.slice.call(arguments, 1);
const filter = _filter[name]; const filter = _filter[name];
_filterChain.push({ func: filter, args }); _filterChain.push({ func: filter, args });
}; };
@ -90,44 +80,13 @@ const GLImageFilter = function (params) {
_filterChain = []; _filterChain = [];
}; };
this.apply = function (image) {
_resize(image.width, image.height);
_drawCount = 0;
// Create the texture for the input image if we haven't yet
if (!_sourceTexture) _sourceTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, _sourceTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
// No filters? Just draw
if (_filterChain.length === 0) {
// const program = _compileShader(SHADER.FRAGMENT_IDENTITY);
_draw();
return _canvas;
}
for (let i = 0; i < _filterChain.length; i++) {
_lastInChain = (i === _filterChain.length - 1);
const f = _filterChain[i];
f.func.apply(this, f.args || []);
}
return _canvas;
};
const _resize = function (width, height) { const _resize = function (width, height) {
// Same width/height? Nothing to do here // Same width/height? Nothing to do here
if (width === _width && height === _height) { return; } if (width === _width && height === _height) { return; }
_canvas.width = width; _canvas.width = width;
_width = width; _width = width;
_canvas.height = height; _canvas.height = height;
_height = height; _height = height;
// Create the context if we don't have it yet // Create the context if we don't have it yet
if (!_vertexBuffer) { if (!_vertexBuffer) {
// Create the vertex buffer for the two triangles [x, y, u, v] * 6 // Create the vertex buffer for the two triangles [x, y, u, v] * 6
@ -138,53 +97,43 @@ const GLImageFilter = function (params) {
// eslint-disable-next-line no-unused-expressions // eslint-disable-next-line no-unused-expressions
(_vertexBuffer = gl.createBuffer(), gl.bindBuffer(gl.ARRAY_BUFFER, _vertexBuffer)); (_vertexBuffer = gl.createBuffer(), gl.bindBuffer(gl.ARRAY_BUFFER, _vertexBuffer));
gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW); gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
// Note sure if this is a good idea; at least it makes texture loading // Note sure if this is a good idea; at least it makes texture loading
// in Ejecta instant. // in Ejecta instant.
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
} }
gl.viewport(0, 0, _width, _height); gl.viewport(0, 0, _width, _height);
// Delete old temp framebuffers // Delete old temp framebuffers
_tempFramebuffers = [null, null]; _tempFramebuffers = [null, null];
}; };
const _createFramebufferTexture = function (width, height) {
const fbo = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
const renderbuffer = gl.createRenderbuffer();
gl.bindRenderbuffer(gl.RENDERBUFFER, renderbuffer);
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
gl.bindTexture(gl.TEXTURE_2D, null);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
return { fbo, texture };
};
const _getTempFramebuffer = function (index) { const _getTempFramebuffer = function (index) {
// @ts-ignore // @ts-ignore
_tempFramebuffers[index] = _tempFramebuffers[index] || _createFramebufferTexture(_width, _height); _tempFramebuffers[index] = _tempFramebuffers[index] || _createFramebufferTexture(_width, _height);
return _tempFramebuffers[index]; return _tempFramebuffers[index];
}; };
const _createFramebufferTexture = function (width, height) {
const fbo = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
const renderbuffer = gl.createRenderbuffer();
gl.bindRenderbuffer(gl.RENDERBUFFER, renderbuffer);
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
gl.bindTexture(gl.TEXTURE_2D, null);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
return { fbo, texture };
};
const _draw = function (flags = null) { const _draw = function (flags = null) {
let source = null; let source = null;
let target = null; let target = null;
let flipY = false; let flipY = false;
// Set up the source // Set up the source
if (_drawCount === 0) { if (_drawCount === 0) {
// First draw call - use the source texture // First draw call - use the source texture
@ -195,7 +144,6 @@ const GLImageFilter = function (params) {
source = _getTempFramebuffer(_currentFramebufferIndex)?.texture; source = _getTempFramebuffer(_currentFramebufferIndex)?.texture;
} }
_drawCount++; _drawCount++;
// Set up the target // Set up the target
if (_lastInChain && !(flags & DRAW.INTERMEDIATE)) { if (_lastInChain && !(flags & DRAW.INTERMEDIATE)) {
// Last filter in our chain - draw directly to the WebGL Canvas. We may // Last filter in our chain - draw directly to the WebGL Canvas. We may
@ -208,67 +156,78 @@ const GLImageFilter = function (params) {
// @ts-ignore // @ts-ignore
target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo; target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo;
} }
// Bind the source and target and draw the two triangles // Bind the source and target and draw the two triangles
gl.bindTexture(gl.TEXTURE_2D, source); gl.bindTexture(gl.TEXTURE_2D, source);
gl.bindFramebuffer(gl.FRAMEBUFFER, target); gl.bindFramebuffer(gl.FRAMEBUFFER, target);
gl.uniform1f(_currentProgram.uniform.flipY, (flipY ? -1 : 1)); gl.uniform1f(_currentProgram.uniform.flipY, (flipY ? -1 : 1));
gl.drawArrays(gl.TRIANGLES, 0, 6); gl.drawArrays(gl.TRIANGLES, 0, 6);
}; };
this.apply = function (image) {
_resize(image.width, image.height);
_drawCount = 0;
// Create the texture for the input image if we haven't yet
if (!_sourceTexture) _sourceTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, _sourceTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
// No filters? Just draw
if (_filterChain.length === 0) {
// const program = _compileShader(SHADER.FRAGMENT_IDENTITY);
_draw();
return _canvas;
}
for (let i = 0; i < _filterChain.length; i++) {
_lastInChain = (i === _filterChain.length - 1);
const f = _filterChain[i];
f.func.apply(this, f.args || []);
}
return _canvas;
};
const _compileShader = function (fragmentSource) { const _compileShader = function (fragmentSource) {
if (_shaderProgramCache[fragmentSource]) { if (_shaderProgramCache[fragmentSource]) {
_currentProgram = _shaderProgramCache[fragmentSource]; _currentProgram = _shaderProgramCache[fragmentSource];
gl.useProgram(_currentProgram.id); gl.useProgram(_currentProgram.id);
return _currentProgram; return _currentProgram;
} }
// Compile shaders // Compile shaders
_currentProgram = new GLProgram(gl, SHADER.VERTEX_IDENTITY, fragmentSource); const SHADER = {};
const floatSize = Float32Array.BYTES_PER_ELEMENT;
const vertSize = 4 * floatSize;
gl.enableVertexAttribArray(_currentProgram.attribute.pos);
gl.vertexAttribPointer(_currentProgram.attribute.pos, 2, gl.FLOAT, false, vertSize, 0 * floatSize);
gl.enableVertexAttribArray(_currentProgram.attribute.uv);
gl.vertexAttribPointer(_currentProgram.attribute.uv, 2, gl.FLOAT, false, vertSize, 2 * floatSize);
_shaderProgramCache[fragmentSource] = _currentProgram;
return _currentProgram;
};
let DRAW = { INTERMEDIATE: 1 };
let SHADER = {};
SHADER.VERTEX_IDENTITY = [ SHADER.VERTEX_IDENTITY = [
'precision highp float;', 'precision highp float;',
'attribute vec2 pos;', 'attribute vec2 pos;',
'attribute vec2 uv;', 'attribute vec2 uv;',
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform float flipY;', 'uniform float flipY;',
'void main(void) {', 'void main(void) {',
'vUv = uv;', 'vUv = uv;',
'gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.);', 'gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.);',
'}', '}',
].join('\n'); ].join('\n');
SHADER.FRAGMENT_IDENTITY = [ SHADER.FRAGMENT_IDENTITY = [
'precision highp float;', 'precision highp float;',
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'void main(void) {', 'void main(void) {',
'gl_FragColor = texture2D(texture, vUv);', 'gl_FragColor = texture2D(texture, vUv);',
'}', '}',
].join('\n'); ].join('\n');
_currentProgram = new GLProgram(gl, SHADER.VERTEX_IDENTITY, fragmentSource);
let _filter = {}; const floatSize = Float32Array.BYTES_PER_ELEMENT;
const vertSize = 4 * floatSize;
gl.enableVertexAttribArray(_currentProgram.attribute.pos);
gl.vertexAttribPointer(_currentProgram.attribute.pos, 2, gl.FLOAT, false, vertSize, 0 * floatSize);
gl.enableVertexAttribArray(_currentProgram.attribute.uv);
gl.vertexAttribPointer(_currentProgram.attribute.uv, 2, gl.FLOAT, false, vertSize, 2 * floatSize);
_shaderProgramCache[fragmentSource] = _currentProgram;
return _currentProgram;
};
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Color Matrix Filter // Color Matrix Filter
_filter.colorMatrix = function (matrix) { _filter.colorMatrix = function (matrix) {
// Create a Float32 Array and normalize the offset component to 0-1 // Create a Float32 Array and normalize the offset component to 0-1
const m = new Float32Array(matrix); const m = new Float32Array(matrix);
@ -276,24 +235,20 @@ const GLImageFilter = function (params) {
m[9] /= 255; m[9] /= 255;
m[14] /= 255; m[14] /= 255;
m[19] /= 255; m[19] /= 255;
// Can we ignore the alpha value? Makes things a bit faster. // Can we ignore the alpha value? Makes things a bit faster.
const shader = (m[18] === 1 && m[3] === 0 && m[8] === 0 && m[13] === 0 && m[15] === 0 && m[16] === 0 && m[17] === 0 && m[19] === 0) const shader = (m[18] === 1 && m[3] === 0 && m[8] === 0 && m[13] === 0 && m[15] === 0 && m[16] === 0 && m[17] === 0 && m[19] === 0)
? _filter.colorMatrix.SHADER.WITHOUT_ALPHA ? _filter.colorMatrix.SHADER.WITHOUT_ALPHA
: _filter.colorMatrix.SHADER.WITH_ALPHA; : _filter.colorMatrix.SHADER.WITH_ALPHA;
const program = _compileShader(shader); const program = _compileShader(shader);
gl.uniform1fv(program.uniform.m, m); gl.uniform1fv(program.uniform.m, m);
_draw(); _draw();
}; };
_filter.colorMatrix.SHADER = {}; _filter.colorMatrix.SHADER = {};
_filter.colorMatrix.SHADER.WITH_ALPHA = [ _filter.colorMatrix.SHADER.WITH_ALPHA = [
'precision highp float;', 'precision highp float;',
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'uniform float m[20];', 'uniform float m[20];',
'void main(void) {', 'void main(void) {',
'vec4 c = texture2D(texture, vUv);', 'vec4 c = texture2D(texture, vUv);',
'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[3] * c.a + m[4];', 'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[3] * c.a + m[4];',
@ -307,7 +262,6 @@ const GLImageFilter = function (params) {
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'uniform float m[20];', 'uniform float m[20];',
'void main(void) {', 'void main(void) {',
'vec4 c = texture2D(texture, vUv);', 'vec4 c = texture2D(texture, vUv);',
'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[4];', 'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[4];',
@ -448,12 +402,10 @@ const GLImageFilter = function (params) {
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Convolution Filter // Convolution Filter
_filter.convolution = function (matrix) { _filter.convolution = function (matrix) {
const m = new Float32Array(matrix); const m = new Float32Array(matrix);
const pixelSizeX = 1 / _width; const pixelSizeX = 1 / _width;
const pixelSizeY = 1 / _height; const pixelSizeY = 1 / _height;
const program = _compileShader(_filter.convolution.SHADER); const program = _compileShader(_filter.convolution.SHADER);
gl.uniform1fv(program.uniform.m, m); gl.uniform1fv(program.uniform.m, m);
gl.uniform2f(program.uniform.px, pixelSizeX, pixelSizeY); gl.uniform2f(program.uniform.px, pixelSizeX, pixelSizeY);
@ -466,20 +418,16 @@ const GLImageFilter = function (params) {
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'uniform vec2 px;', 'uniform vec2 px;',
'uniform float m[9];', 'uniform float m[9];',
'void main(void) {', 'void main(void) {',
'vec4 c11 = texture2D(texture, vUv - px);', // top left 'vec4 c11 = texture2D(texture, vUv - px);', // top left
'vec4 c12 = texture2D(texture, vec2(vUv.x, vUv.y - px.y));', // top center 'vec4 c12 = texture2D(texture, vec2(vUv.x, vUv.y - px.y));', // top center
'vec4 c13 = texture2D(texture, vec2(vUv.x + px.x, vUv.y - px.y));', // top right 'vec4 c13 = texture2D(texture, vec2(vUv.x + px.x, vUv.y - px.y));', // top right
'vec4 c21 = texture2D(texture, vec2(vUv.x - px.x, vUv.y) );', // mid left 'vec4 c21 = texture2D(texture, vec2(vUv.x - px.x, vUv.y) );', // mid left
'vec4 c22 = texture2D(texture, vUv);', // mid center 'vec4 c22 = texture2D(texture, vUv);', // mid center
'vec4 c23 = texture2D(texture, vec2(vUv.x + px.x, vUv.y) );', // mid right 'vec4 c23 = texture2D(texture, vec2(vUv.x + px.x, vUv.y) );', // mid right
'vec4 c31 = texture2D(texture, vec2(vUv.x - px.x, vUv.y + px.y) );', // bottom left 'vec4 c31 = texture2D(texture, vec2(vUv.x - px.x, vUv.y + px.y) );', // bottom left
'vec4 c32 = texture2D(texture, vec2(vUv.x, vUv.y + px.y) );', // bottom center 'vec4 c32 = texture2D(texture, vec2(vUv.x, vUv.y + px.y) );', // bottom center
'vec4 c33 = texture2D(texture, vUv + px );', // bottom right 'vec4 c33 = texture2D(texture, vUv + px );', // bottom right
'gl_FragColor = ', 'gl_FragColor = ',
'c11 * m[0] + c12 * m[1] + c22 * m[2] +', 'c11 * m[0] + c12 * m[1] + c22 * m[2] +',
'c21 * m[3] + c22 * m[4] + c23 * m[5] +', 'c21 * m[3] + c22 * m[4] + c23 * m[5] +',
@ -532,17 +480,13 @@ const GLImageFilter = function (params) {
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Blur Filter // Blur Filter
_filter.blur = function (size) { _filter.blur = function (size) {
const blurSizeX = (size / 7) / _width; const blurSizeX = (size / 7) / _width;
const blurSizeY = (size / 7) / _height; const blurSizeY = (size / 7) / _height;
const program = _compileShader(_filter.blur.SHADER); const program = _compileShader(_filter.blur.SHADER);
// Vertical // Vertical
gl.uniform2f(program.uniform.px, 0, blurSizeY); gl.uniform2f(program.uniform.px, 0, blurSizeY);
_draw(DRAW.INTERMEDIATE); _draw(DRAW.INTERMEDIATE);
// Horizontal // Horizontal
gl.uniform2f(program.uniform.px, blurSizeX, 0); gl.uniform2f(program.uniform.px, blurSizeX, 0);
_draw(); _draw();
@ -553,7 +497,6 @@ const GLImageFilter = function (params) {
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'uniform vec2 px;', 'uniform vec2 px;',
'void main(void) {', 'void main(void) {',
'gl_FragColor = vec4(0.0);', 'gl_FragColor = vec4(0.0);',
'gl_FragColor += texture2D(texture, vUv + vec2(-7.0*px.x, -7.0*px.y))*0.0044299121055113265;', 'gl_FragColor += texture2D(texture, vUv + vec2(-7.0*px.x, -7.0*px.y))*0.0044299121055113265;',
@ -576,13 +519,10 @@ const GLImageFilter = function (params) {
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Pixelate Filter // Pixelate Filter
_filter.pixelate = function (size) { _filter.pixelate = function (size) {
const blurSizeX = (size) / _width; const blurSizeX = (size) / _width;
const blurSizeY = (size) / _height; const blurSizeY = (size) / _height;
const program = _compileShader(_filter.pixelate.SHADER); const program = _compileShader(_filter.pixelate.SHADER);
// Horizontal // Horizontal
gl.uniform2f(program.uniform.size, blurSizeX, blurSizeY); gl.uniform2f(program.uniform.size, blurSizeX, blurSizeY);
_draw(); _draw();
@ -593,17 +533,13 @@ const GLImageFilter = function (params) {
'varying vec2 vUv;', 'varying vec2 vUv;',
'uniform vec2 size;', 'uniform vec2 size;',
'uniform sampler2D texture;', 'uniform sampler2D texture;',
'vec2 pixelate(vec2 coord, vec2 size) {', 'vec2 pixelate(vec2 coord, vec2 size) {',
'return floor( coord / size ) * size;', 'return floor( coord / size ) * size;',
'}', '}',
'void main(void) {', 'void main(void) {',
'gl_FragColor = vec4(0.0);', 'gl_FragColor = vec4(0.0);',
'vec2 coord = pixelate(vUv, size);', 'vec2 coord = pixelate(vUv, size);',
'gl_FragColor += texture2D(texture, coord);', 'gl_FragColor += texture2D(texture, coord);',
'}', '}',
].join('\n'); ].join('\n');
}; }
exports.GLImageFilter = GLImageFilter;

1
types/imagefx.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export declare function GLImageFilter(params: any): void;