mirror of https://github.com/vladmandic/human
update imagefx
Vladimir Mandic
parent
34f4bc5ee4
commit
5c336f60d3
28
README.md
28
README.md
|
|
@ -1,6 +1,16 @@
|
|||
# Human Library
|
||||
|
||||
## 3D Face Detection, Face Embedding & Recognition, Body Pose Tracking, Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction & Gesture Recognition
|
||||
### 3D Face Detection, Face Embedding & Recognition,
|
||||
### Body Pose Tracking, Hand & Finger Tracking,
|
||||
### Iris Analysis, Age & Gender & Emotion Prediction
|
||||
### & Gesture Recognition
|
||||
|
||||
<br>
|
||||
|
||||
Native JavaScript module using TensorFlow/JS Machine Learning library
|
||||
Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows and Linux
|
||||
- Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends
|
||||
- NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu*
|
||||
|
||||
<br>
|
||||
|
||||
|
|
@ -39,28 +49,17 @@
|
|||
|
||||
<br>
|
||||
|
||||
Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows and Linux
|
||||
- Browser/WebWorker: Compatible with *CPU*, *WebGL*, *WASM* and *WebGPU* backends
|
||||
- NodeJS: Compatible with software *tfjs-node* and CUDA accelerated backends *tfjs-node-gpu*
|
||||
- (and maybe with React-Native as it doesn't use any DOM objects)
|
||||
|
||||
<br>
|
||||
|
||||
*See [issues](https://github.com/vladmandic/human/issues?q=) and [discussions](https://github.com/vladmandic/human/discussions) for list of known limitations and planned enhancements*
|
||||
|
||||
*Suggestions are welcome!*
|
||||
|
||||
<br>
|
||||
<hr>
|
||||
<br>
|
||||
<br><hr><br>
|
||||
|
||||
## Options ##
|
||||
|
||||
|
||||
|
||||
<br>
|
||||
<hr>
|
||||
<br>
|
||||
<br><hr><br>
|
||||
|
||||
## Examples
|
||||
|
||||
|
|
@ -75,4 +74,3 @@ Compatible with *Browser*, *WebWorker* and *NodeJS* execution on both Windows an
|
|||
**Using webcam:**
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import Human from '../dist/human.esm.js';
|
||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||
// import * as tf from '@tensorflow/tfjs';
|
||||
// import Human from '../dist/human.esm-nobundle.js';
|
||||
|
||||
import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
|
||||
|
||||
import draw from './draw.js';
|
||||
import Menu from './menu.js';
|
||||
import GLBench from './gl-bench.js';
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
"main": "dist/human.node.js",
|
||||
"module": "dist/human.esm.js",
|
||||
"browser": "dist/human.esm.js",
|
||||
"types": "types/human.d.ts",
|
||||
"author": "Vladimir Mandic <mandic00@live.com>",
|
||||
"bugs": {
|
||||
"url": "https://github.com/vladmandic/human/issues"
|
||||
|
|
@ -32,13 +33,13 @@
|
|||
"@tensorflow/tfjs-layers": "^3.1.0",
|
||||
"@tensorflow/tfjs-node": "^3.1.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.1.0",
|
||||
"@types/node": "^14.14.28",
|
||||
"@types/node": "^14.14.30",
|
||||
"@typescript-eslint/eslint-plugin": "^4.15.1",
|
||||
"@typescript-eslint/parser": "^4.15.1",
|
||||
"@vladmandic/pilogger": "^0.2.14",
|
||||
"chokidar": "^3.5.1",
|
||||
"dayjs": "^1.10.4",
|
||||
"esbuild": "^0.8.46",
|
||||
"esbuild": "^0.8.49",
|
||||
"eslint": "^7.20.0",
|
||||
"eslint-config-airbnb-base": "^14.2.1",
|
||||
"eslint-plugin-import": "^2.22.1",
|
||||
|
|
@ -47,9 +48,9 @@
|
|||
"eslint-plugin-promise": "^4.3.1",
|
||||
"rimraf": "^3.0.2",
|
||||
"seedrandom": "^3.0.5",
|
||||
"simple-git": "^2.35.0",
|
||||
"simple-git": "^2.35.1",
|
||||
"tslib": "^2.1.0",
|
||||
"typescript": "^4.3.0-dev.20210217"
|
||||
"typescript": "^4.3.0-dev.20210219"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node --trace-warnings --unhandled-rejections=strict --trace-uncaught --no-deprecation src/node.js",
|
||||
|
|
|
|||
44
src/human.ts
44
src/human.ts
|
|
@ -101,7 +101,7 @@ class Human {
|
|||
// helper function: measure tensor leak
|
||||
analyze(...msg) {
|
||||
if (!this.analyzeMemoryLeaks) return;
|
||||
const current = tf.engine().state.numTensors;
|
||||
const current = this.tf.engine().state.numTensors;
|
||||
const previous = this.numTensors;
|
||||
this.numTensors = current;
|
||||
const leaked = current - previous;
|
||||
|
|
@ -112,11 +112,11 @@ class Human {
|
|||
sanity(input) {
|
||||
if (!this.checkSanity) return null;
|
||||
if (!input) return 'input is not defined';
|
||||
if (tf.ENV.flags.IS_NODE && !(input instanceof tf.Tensor)) {
|
||||
if (this.tf.ENV.flags.IS_NODE && !(input instanceof this.tf.Tensor)) {
|
||||
return 'input must be a tensor';
|
||||
}
|
||||
try {
|
||||
tf.getBackend();
|
||||
this.tf.getBackend();
|
||||
} catch {
|
||||
return 'backend not loaded';
|
||||
}
|
||||
|
|
@ -135,11 +135,11 @@ class Human {
|
|||
if (userConfig) this.config = mergeDeep(this.config, userConfig);
|
||||
|
||||
if (this.firstRun) {
|
||||
log(`version: ${this.version} TensorFlow/JS version: ${tf.version_core}`);
|
||||
log(`version: ${this.version} TensorFlow/JS version: ${this.tf.version_core}`);
|
||||
await this.checkBackend(true);
|
||||
if (tf.ENV.flags.IS_BROWSER) {
|
||||
if (this.tf.ENV.flags.IS_BROWSER) {
|
||||
log('configuration:', this.config);
|
||||
log('tf flags:', tf.ENV.flags);
|
||||
log('tf flags:', this.tf.ENV.flags);
|
||||
}
|
||||
}
|
||||
const face = this.config.face.detector.modelPath.includes('faceboxes') ? faceboxes : facemesh;
|
||||
|
|
@ -172,7 +172,7 @@ class Human {
|
|||
}
|
||||
|
||||
if (this.firstRun) {
|
||||
log('tf engine state:', tf.engine().state.numBytes, 'bytes', tf.engine().state.numTensors, 'tensors');
|
||||
log('tf engine state:', this.tf.engine().state.numBytes, 'bytes', this.tf.engine().state.numTensors, 'tensors');
|
||||
this.firstRun = false;
|
||||
}
|
||||
|
||||
|
|
@ -182,7 +182,7 @@ class Human {
|
|||
|
||||
// check if backend needs initialization if it changed
|
||||
async checkBackend(force = false) {
|
||||
if (this.config.backend && (this.config.backend !== '') && force || (tf.getBackend() !== this.config.backend)) {
|
||||
if (this.config.backend && (this.config.backend !== '') && force || (this.tf.getBackend() !== this.config.backend)) {
|
||||
const timeStamp = now();
|
||||
this.state = 'backend';
|
||||
/* force backend reload
|
||||
|
|
@ -199,32 +199,32 @@ class Human {
|
|||
|
||||
if (this.config.backend === 'wasm') {
|
||||
log('settings wasm path:', this.config.wasmPath);
|
||||
tf.setWasmPaths(this.config.wasmPath);
|
||||
const simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
|
||||
this.tf.setWasmPaths(this.config.wasmPath);
|
||||
const simd = await this.tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
|
||||
if (!simd) log('warning: wasm simd support is not enabled');
|
||||
}
|
||||
|
||||
if (this.config.backend === 'humangl') backend.register();
|
||||
try {
|
||||
await tf.setBackend(this.config.backend);
|
||||
await this.tf.setBackend(this.config.backend);
|
||||
} catch (err) {
|
||||
log('error: cannot set backend:', this.config.backend, err);
|
||||
}
|
||||
tf.enableProdMode();
|
||||
this.tf.enableProdMode();
|
||||
/* debug mode is really too mcuh
|
||||
tf.enableDebugMode();
|
||||
*/
|
||||
if (tf.getBackend() === 'webgl') {
|
||||
if (this.tf.getBackend() === 'webgl') {
|
||||
if (this.config.deallocate) {
|
||||
log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', this.config.deallocate);
|
||||
tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
|
||||
this.tf.ENV.set('WEBGL_DELETE_TEXTURE_THRESHOLD', this.config.deallocate ? 0 : -1);
|
||||
}
|
||||
tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
|
||||
tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
|
||||
const gl = await tf.backend().getGPGPUContext().gl;
|
||||
this.tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true);
|
||||
this.tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', true);
|
||||
const gl = await this.tf.backend().getGPGPUContext().gl;
|
||||
log(`gl version:${gl.getParameter(gl.VERSION)} renderer:${gl.getParameter(gl.RENDERER)}`);
|
||||
}
|
||||
await tf.ready();
|
||||
await this.tf.ready();
|
||||
this.perf.backend = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
}
|
||||
|
|
@ -384,7 +384,7 @@ class Human {
|
|||
// load models if enabled
|
||||
await this.load();
|
||||
|
||||
if (this.config.scoped) tf.engine().startScope();
|
||||
if (this.config.scoped) this.tf.engine().startScope();
|
||||
this.analyze('Start Scope:');
|
||||
|
||||
timeStamp = now();
|
||||
|
|
@ -440,7 +440,7 @@ class Human {
|
|||
}
|
||||
process.tensor.dispose();
|
||||
|
||||
if (this.config.scoped) tf.engine().endScope();
|
||||
if (this.config.scoped) this.tf.engine().endScope();
|
||||
this.analyze('End Scope:');
|
||||
|
||||
let gestureRes = [];
|
||||
|
|
@ -512,10 +512,10 @@ class Human {
|
|||
// @ts-ignore
|
||||
const data = tf.node.decodeJpeg(img); // tf.node is only defined when compiling for nodejs
|
||||
const expanded = data.expandDims(0);
|
||||
tf.dispose(data);
|
||||
this.tf.dispose(data);
|
||||
// log('Input:', expanded);
|
||||
const res = await this.detect(expanded, this.config);
|
||||
tf.dispose(expanded);
|
||||
this.tf.dispose(expanded);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ export function process(input, config) {
|
|||
*/
|
||||
} else {
|
||||
outCanvas = inCanvas;
|
||||
if (this.fx) this.fx = null;
|
||||
}
|
||||
let pixels;
|
||||
if (outCanvas.data) {
|
||||
|
|
|
|||
208
src/imagefx.js
208
src/imagefx.js
|
|
@ -1,11 +1,10 @@
|
|||
/* eslint-disable no-use-before-define */
|
||||
/*
|
||||
WebGLImageFilter - MIT Licensed
|
||||
2013, Dominic Szablewski - phoboslab.org
|
||||
<https://github.com/phoboslab/WebGLImageFilter>
|
||||
*/
|
||||
|
||||
const GLProgram = function (gl, vertexSource, fragmentSource) {
|
||||
function GLProgram(gl, vertexSource, fragmentSource) {
|
||||
const _collect = function (source, prefix, collection) {
|
||||
const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig');
|
||||
source.replace(r, (match, name) => {
|
||||
|
|
@ -18,7 +17,6 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
|
|||
const shader = gl.createShader(type);
|
||||
gl.shaderSource(shader, source);
|
||||
gl.compileShader(shader);
|
||||
|
||||
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
||||
// @ts-ignore
|
||||
throw new Error('Filter: GL compile failed', gl.getShaderInfoLog(shader));
|
||||
|
|
@ -28,10 +26,8 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
|
|||
|
||||
this.uniform = {};
|
||||
this.attribute = {};
|
||||
|
||||
const _vsh = _compile(vertexSource, gl.VERTEX_SHADER);
|
||||
const _fsh = _compile(fragmentSource, gl.FRAGMENT_SHADER);
|
||||
|
||||
this.id = gl.createProgram();
|
||||
gl.attachShader(this.id, _vsh);
|
||||
gl.attachShader(this.id, _fsh);
|
||||
|
|
@ -43,22 +39,17 @@ const GLProgram = function (gl, vertexSource, fragmentSource) {
|
|||
}
|
||||
|
||||
gl.useProgram(this.id);
|
||||
|
||||
// Collect attributes
|
||||
_collect(vertexSource, 'attribute', this.attribute);
|
||||
for (const a in this.attribute) {
|
||||
this.attribute[a] = gl.getAttribLocation(this.id, a);
|
||||
}
|
||||
|
||||
for (const a in this.attribute) this.attribute[a] = gl.getAttribLocation(this.id, a);
|
||||
// Collect uniforms
|
||||
_collect(vertexSource, 'uniform', this.uniform);
|
||||
_collect(fragmentSource, 'uniform', this.uniform);
|
||||
for (const u in this.uniform) {
|
||||
this.uniform[u] = gl.getUniformLocation(this.id, u);
|
||||
}
|
||||
};
|
||||
for (const u in this.uniform) this.uniform[u] = gl.getUniformLocation(this.id, u);
|
||||
}
|
||||
|
||||
const GLImageFilter = function (params) {
|
||||
// export const GLImageFilter = function (params) {
|
||||
export function GLImageFilter(params) {
|
||||
if (!params) params = { };
|
||||
let _drawCount = 0;
|
||||
let _sourceTexture = null;
|
||||
|
|
@ -70,11 +61,11 @@ const GLImageFilter = function (params) {
|
|||
let _height = -1;
|
||||
let _vertexBuffer = null;
|
||||
let _currentProgram = null;
|
||||
const _filter = {};
|
||||
const _canvas = params.canvas || document.createElement('canvas');
|
||||
|
||||
// key is the shader program source, value is the compiled program
|
||||
const _shaderProgramCache = { };
|
||||
|
||||
const DRAW = { INTERMEDIATE: 1 };
|
||||
const gl = _canvas.getContext('webgl');
|
||||
if (!gl) throw new Error('Filter: getContext() failed');
|
||||
|
||||
|
|
@ -82,7 +73,6 @@ const GLImageFilter = function (params) {
|
|||
// eslint-disable-next-line prefer-rest-params
|
||||
const args = Array.prototype.slice.call(arguments, 1);
|
||||
const filter = _filter[name];
|
||||
|
||||
_filterChain.push({ func: filter, args });
|
||||
};
|
||||
|
||||
|
|
@ -90,44 +80,13 @@ const GLImageFilter = function (params) {
|
|||
_filterChain = [];
|
||||
};
|
||||
|
||||
this.apply = function (image) {
|
||||
_resize(image.width, image.height);
|
||||
_drawCount = 0;
|
||||
|
||||
// Create the texture for the input image if we haven't yet
|
||||
if (!_sourceTexture) _sourceTexture = gl.createTexture();
|
||||
gl.bindTexture(gl.TEXTURE_2D, _sourceTexture);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
|
||||
|
||||
// No filters? Just draw
|
||||
if (_filterChain.length === 0) {
|
||||
// const program = _compileShader(SHADER.FRAGMENT_IDENTITY);
|
||||
_draw();
|
||||
return _canvas;
|
||||
}
|
||||
|
||||
for (let i = 0; i < _filterChain.length; i++) {
|
||||
_lastInChain = (i === _filterChain.length - 1);
|
||||
const f = _filterChain[i];
|
||||
f.func.apply(this, f.args || []);
|
||||
}
|
||||
|
||||
return _canvas;
|
||||
};
|
||||
|
||||
const _resize = function (width, height) {
|
||||
// Same width/height? Nothing to do here
|
||||
if (width === _width && height === _height) { return; }
|
||||
|
||||
_canvas.width = width;
|
||||
_width = width;
|
||||
_canvas.height = height;
|
||||
_height = height;
|
||||
|
||||
// Create the context if we don't have it yet
|
||||
if (!_vertexBuffer) {
|
||||
// Create the vertex buffer for the two triangles [x, y, u, v] * 6
|
||||
|
|
@ -138,53 +97,43 @@ const GLImageFilter = function (params) {
|
|||
// eslint-disable-next-line no-unused-expressions
|
||||
(_vertexBuffer = gl.createBuffer(), gl.bindBuffer(gl.ARRAY_BUFFER, _vertexBuffer));
|
||||
gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);
|
||||
|
||||
// Note sure if this is a good idea; at least it makes texture loading
|
||||
// in Ejecta instant.
|
||||
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);
|
||||
}
|
||||
|
||||
gl.viewport(0, 0, _width, _height);
|
||||
|
||||
// Delete old temp framebuffers
|
||||
_tempFramebuffers = [null, null];
|
||||
};
|
||||
|
||||
const _createFramebufferTexture = function (width, height) {
|
||||
const fbo = gl.createFramebuffer();
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
|
||||
const renderbuffer = gl.createRenderbuffer();
|
||||
gl.bindRenderbuffer(gl.RENDERBUFFER, renderbuffer);
|
||||
const texture = gl.createTexture();
|
||||
gl.bindTexture(gl.TEXTURE_2D, texture);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
|
||||
gl.bindTexture(gl.TEXTURE_2D, null);
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
||||
return { fbo, texture };
|
||||
};
|
||||
|
||||
const _getTempFramebuffer = function (index) {
|
||||
// @ts-ignore
|
||||
_tempFramebuffers[index] = _tempFramebuffers[index] || _createFramebufferTexture(_width, _height);
|
||||
return _tempFramebuffers[index];
|
||||
};
|
||||
|
||||
const _createFramebufferTexture = function (width, height) {
|
||||
const fbo = gl.createFramebuffer();
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
|
||||
|
||||
const renderbuffer = gl.createRenderbuffer();
|
||||
gl.bindRenderbuffer(gl.RENDERBUFFER, renderbuffer);
|
||||
|
||||
const texture = gl.createTexture();
|
||||
gl.bindTexture(gl.TEXTURE_2D, texture);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
|
||||
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
|
||||
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
|
||||
|
||||
gl.bindTexture(gl.TEXTURE_2D, null);
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
||||
|
||||
return { fbo, texture };
|
||||
};
|
||||
|
||||
const _draw = function (flags = null) {
|
||||
let source = null;
|
||||
let target = null;
|
||||
let flipY = false;
|
||||
|
||||
// Set up the source
|
||||
if (_drawCount === 0) {
|
||||
// First draw call - use the source texture
|
||||
|
|
@ -195,7 +144,6 @@ const GLImageFilter = function (params) {
|
|||
source = _getTempFramebuffer(_currentFramebufferIndex)?.texture;
|
||||
}
|
||||
_drawCount++;
|
||||
|
||||
// Set up the target
|
||||
if (_lastInChain && !(flags & DRAW.INTERMEDIATE)) {
|
||||
// Last filter in our chain - draw directly to the WebGL Canvas. We may
|
||||
|
|
@ -208,67 +156,78 @@ const GLImageFilter = function (params) {
|
|||
// @ts-ignore
|
||||
target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo;
|
||||
}
|
||||
|
||||
// Bind the source and target and draw the two triangles
|
||||
gl.bindTexture(gl.TEXTURE_2D, source);
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, target);
|
||||
|
||||
gl.uniform1f(_currentProgram.uniform.flipY, (flipY ? -1 : 1));
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
};
|
||||
|
||||
this.apply = function (image) {
|
||||
_resize(image.width, image.height);
|
||||
_drawCount = 0;
|
||||
// Create the texture for the input image if we haven't yet
|
||||
if (!_sourceTexture) _sourceTexture = gl.createTexture();
|
||||
gl.bindTexture(gl.TEXTURE_2D, _sourceTexture);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image);
|
||||
// No filters? Just draw
|
||||
if (_filterChain.length === 0) {
|
||||
// const program = _compileShader(SHADER.FRAGMENT_IDENTITY);
|
||||
_draw();
|
||||
return _canvas;
|
||||
}
|
||||
for (let i = 0; i < _filterChain.length; i++) {
|
||||
_lastInChain = (i === _filterChain.length - 1);
|
||||
const f = _filterChain[i];
|
||||
f.func.apply(this, f.args || []);
|
||||
}
|
||||
return _canvas;
|
||||
};
|
||||
|
||||
const _compileShader = function (fragmentSource) {
|
||||
if (_shaderProgramCache[fragmentSource]) {
|
||||
_currentProgram = _shaderProgramCache[fragmentSource];
|
||||
gl.useProgram(_currentProgram.id);
|
||||
return _currentProgram;
|
||||
}
|
||||
|
||||
// Compile shaders
|
||||
const SHADER = {};
|
||||
SHADER.VERTEX_IDENTITY = [
|
||||
'precision highp float;',
|
||||
'attribute vec2 pos;',
|
||||
'attribute vec2 uv;',
|
||||
'varying vec2 vUv;',
|
||||
'uniform float flipY;',
|
||||
'void main(void) {',
|
||||
'vUv = uv;',
|
||||
'gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.);',
|
||||
'}',
|
||||
].join('\n');
|
||||
SHADER.FRAGMENT_IDENTITY = [
|
||||
'precision highp float;',
|
||||
'varying vec2 vUv;',
|
||||
'uniform sampler2D texture;',
|
||||
'void main(void) {',
|
||||
'gl_FragColor = texture2D(texture, vUv);',
|
||||
'}',
|
||||
].join('\n');
|
||||
_currentProgram = new GLProgram(gl, SHADER.VERTEX_IDENTITY, fragmentSource);
|
||||
|
||||
const floatSize = Float32Array.BYTES_PER_ELEMENT;
|
||||
const vertSize = 4 * floatSize;
|
||||
gl.enableVertexAttribArray(_currentProgram.attribute.pos);
|
||||
gl.vertexAttribPointer(_currentProgram.attribute.pos, 2, gl.FLOAT, false, vertSize, 0 * floatSize);
|
||||
gl.enableVertexAttribArray(_currentProgram.attribute.uv);
|
||||
gl.vertexAttribPointer(_currentProgram.attribute.uv, 2, gl.FLOAT, false, vertSize, 2 * floatSize);
|
||||
|
||||
_shaderProgramCache[fragmentSource] = _currentProgram;
|
||||
return _currentProgram;
|
||||
};
|
||||
|
||||
let DRAW = { INTERMEDIATE: 1 };
|
||||
|
||||
let SHADER = {};
|
||||
SHADER.VERTEX_IDENTITY = [
|
||||
'precision highp float;',
|
||||
'attribute vec2 pos;',
|
||||
'attribute vec2 uv;',
|
||||
'varying vec2 vUv;',
|
||||
'uniform float flipY;',
|
||||
|
||||
'void main(void) {',
|
||||
'vUv = uv;',
|
||||
'gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.);',
|
||||
'}',
|
||||
].join('\n');
|
||||
|
||||
SHADER.FRAGMENT_IDENTITY = [
|
||||
'precision highp float;',
|
||||
'varying vec2 vUv;',
|
||||
'uniform sampler2D texture;',
|
||||
|
||||
'void main(void) {',
|
||||
'gl_FragColor = texture2D(texture, vUv);',
|
||||
'}',
|
||||
].join('\n');
|
||||
|
||||
let _filter = {};
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Color Matrix Filter
|
||||
|
||||
_filter.colorMatrix = function (matrix) {
|
||||
// Create a Float32 Array and normalize the offset component to 0-1
|
||||
const m = new Float32Array(matrix);
|
||||
|
|
@ -276,24 +235,20 @@ const GLImageFilter = function (params) {
|
|||
m[9] /= 255;
|
||||
m[14] /= 255;
|
||||
m[19] /= 255;
|
||||
|
||||
// Can we ignore the alpha value? Makes things a bit faster.
|
||||
const shader = (m[18] === 1 && m[3] === 0 && m[8] === 0 && m[13] === 0 && m[15] === 0 && m[16] === 0 && m[17] === 0 && m[19] === 0)
|
||||
? _filter.colorMatrix.SHADER.WITHOUT_ALPHA
|
||||
: _filter.colorMatrix.SHADER.WITH_ALPHA;
|
||||
|
||||
const program = _compileShader(shader);
|
||||
gl.uniform1fv(program.uniform.m, m);
|
||||
_draw();
|
||||
};
|
||||
|
||||
_filter.colorMatrix.SHADER = {};
|
||||
_filter.colorMatrix.SHADER.WITH_ALPHA = [
|
||||
'precision highp float;',
|
||||
'varying vec2 vUv;',
|
||||
'uniform sampler2D texture;',
|
||||
'uniform float m[20];',
|
||||
|
||||
'void main(void) {',
|
||||
'vec4 c = texture2D(texture, vUv);',
|
||||
'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[3] * c.a + m[4];',
|
||||
|
|
@ -307,7 +262,6 @@ const GLImageFilter = function (params) {
|
|||
'varying vec2 vUv;',
|
||||
'uniform sampler2D texture;',
|
||||
'uniform float m[20];',
|
||||
|
||||
'void main(void) {',
|
||||
'vec4 c = texture2D(texture, vUv);',
|
||||
'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[4];',
|
||||
|
|
@ -448,12 +402,10 @@ const GLImageFilter = function (params) {
|
|||
|
||||
// -------------------------------------------------------------------------
|
||||
// Convolution Filter
|
||||
|
||||
_filter.convolution = function (matrix) {
|
||||
const m = new Float32Array(matrix);
|
||||
const pixelSizeX = 1 / _width;
|
||||
const pixelSizeY = 1 / _height;
|
||||
|
||||
const program = _compileShader(_filter.convolution.SHADER);
|
||||
gl.uniform1fv(program.uniform.m, m);
|
||||
gl.uniform2f(program.uniform.px, pixelSizeX, pixelSizeY);
|
||||
|
|
@ -466,20 +418,16 @@ const GLImageFilter = function (params) {
|
|||
'uniform sampler2D texture;',
|
||||
'uniform vec2 px;',
|
||||
'uniform float m[9];',
|
||||
|
||||
'void main(void) {',
|
||||
'vec4 c11 = texture2D(texture, vUv - px);', // top left
|
||||
'vec4 c12 = texture2D(texture, vec2(vUv.x, vUv.y - px.y));', // top center
|
||||
'vec4 c13 = texture2D(texture, vec2(vUv.x + px.x, vUv.y - px.y));', // top right
|
||||
|
||||
'vec4 c21 = texture2D(texture, vec2(vUv.x - px.x, vUv.y) );', // mid left
|
||||
'vec4 c22 = texture2D(texture, vUv);', // mid center
|
||||
'vec4 c23 = texture2D(texture, vec2(vUv.x + px.x, vUv.y) );', // mid right
|
||||
|
||||
'vec4 c31 = texture2D(texture, vec2(vUv.x - px.x, vUv.y + px.y) );', // bottom left
|
||||
'vec4 c32 = texture2D(texture, vec2(vUv.x, vUv.y + px.y) );', // bottom center
|
||||
'vec4 c33 = texture2D(texture, vUv + px );', // bottom right
|
||||
|
||||
'gl_FragColor = ',
|
||||
'c11 * m[0] + c12 * m[1] + c22 * m[2] +',
|
||||
'c21 * m[3] + c22 * m[4] + c23 * m[5] +',
|
||||
|
|
@ -532,17 +480,13 @@ const GLImageFilter = function (params) {
|
|||
|
||||
// -------------------------------------------------------------------------
|
||||
// Blur Filter
|
||||
|
||||
_filter.blur = function (size) {
|
||||
const blurSizeX = (size / 7) / _width;
|
||||
const blurSizeY = (size / 7) / _height;
|
||||
|
||||
const program = _compileShader(_filter.blur.SHADER);
|
||||
|
||||
// Vertical
|
||||
gl.uniform2f(program.uniform.px, 0, blurSizeY);
|
||||
_draw(DRAW.INTERMEDIATE);
|
||||
|
||||
// Horizontal
|
||||
gl.uniform2f(program.uniform.px, blurSizeX, 0);
|
||||
_draw();
|
||||
|
|
@ -553,7 +497,6 @@ const GLImageFilter = function (params) {
|
|||
'varying vec2 vUv;',
|
||||
'uniform sampler2D texture;',
|
||||
'uniform vec2 px;',
|
||||
|
||||
'void main(void) {',
|
||||
'gl_FragColor = vec4(0.0);',
|
||||
'gl_FragColor += texture2D(texture, vUv + vec2(-7.0*px.x, -7.0*px.y))*0.0044299121055113265;',
|
||||
|
|
@ -576,13 +519,10 @@ const GLImageFilter = function (params) {
|
|||
|
||||
// -------------------------------------------------------------------------
|
||||
// Pixelate Filter
|
||||
|
||||
_filter.pixelate = function (size) {
|
||||
const blurSizeX = (size) / _width;
|
||||
const blurSizeY = (size) / _height;
|
||||
|
||||
const program = _compileShader(_filter.pixelate.SHADER);
|
||||
|
||||
// Horizontal
|
||||
gl.uniform2f(program.uniform.size, blurSizeX, blurSizeY);
|
||||
_draw();
|
||||
|
|
@ -593,17 +533,13 @@ const GLImageFilter = function (params) {
|
|||
'varying vec2 vUv;',
|
||||
'uniform vec2 size;',
|
||||
'uniform sampler2D texture;',
|
||||
|
||||
'vec2 pixelate(vec2 coord, vec2 size) {',
|
||||
'return floor( coord / size ) * size;',
|
||||
'}',
|
||||
|
||||
'void main(void) {',
|
||||
'gl_FragColor = vec4(0.0);',
|
||||
'vec2 coord = pixelate(vUv, size);',
|
||||
'gl_FragColor += texture2D(texture, coord);',
|
||||
'}',
|
||||
].join('\n');
|
||||
};
|
||||
|
||||
exports.GLImageFilter = GLImageFilter;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue