From d23fb162a9355397fe89d32faddae9c179f47c06 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 12 Oct 2021 09:48:00 -0400 Subject: [PATCH] update imagefx --- demo/index.js | 19 +- src/image/image.ts | 3 +- src/image/imagefx.ts | 697 ++++++++++++++---------------------- src/image/imagefxshaders.ts | 112 ++++++ 4 files changed, 406 insertions(+), 425 deletions(-) create mode 100644 src/image/imagefxshaders.ts diff --git a/demo/index.js b/demo/index.js index 3030b63b..194846cd 100644 --- a/demo/index.js +++ b/demo/index.js @@ -31,6 +31,9 @@ import jsonView from './helpers/jsonview.js'; let human; let userConfig = { + face: { enabled: false }, + body: { enabled: false }, + hand: { enabled: false }, /* warmup: 'none', backend: 'humangl', @@ -91,7 +94,7 @@ const ui = { autoPlay: false, // start webcam & detection on load // internal variables - exceptionHandler: true, // should capture all unhandled exceptions + exceptionHandler: false, // should capture all unhandled exceptions busy: false, // internal camera busy flag menuWidth: 0, // internal menuHeight: 0, // internal @@ -147,6 +150,10 @@ let worker; let bench; let lastDetectedResult = {}; +// helper function: async pause +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + // helper function: translates json to human readable string function str(...msg) { if (!Array.isArray(msg)) return msg; @@ -225,15 +232,18 @@ async function calcSimmilarity(result) { } const isLive = (input) => { - const videoLive = input.readyState > 2; - const cameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live'; - const live = (videoLive || cameraLive) && (!input.paused); + const isCamera = input.srcObject?.getVideoTracks()[0] && input.srcObject?.getVideoTracks()[0].enabled; + const isVideoLive = input.readyState > 2; + const isCameraLive = input.srcObject?.getVideoTracks()[0].readyState === 'live'; + let live = isCamera ? isCameraLive : isVideoLive; + live = live && !input.paused; return live; }; // draws processed results and starts processing of a next frame let lastDraw = performance.now(); async function drawResults(input) { + // await delay(25); const result = lastDetectedResult; const canvas = document.getElementById('canvas'); @@ -325,6 +335,7 @@ async function drawResults(input) { ui.drawThread = requestAnimationFrame(() => drawResults(input)); } else { cancelAnimationFrame(ui.drawThread); + videoPause(); ui.drawThread = null; } } else { diff --git a/src/image/image.ts b/src/image/image.ts index 7e1be008..49f058c7 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -148,7 +148,7 @@ export function process(input: Input, config: Config, getTensor: boolean = true) if (config.filter.technicolor) fx.addFilter('technicolor'); if (config.filter.polaroid) fx.addFilter('polaroid'); if (config.filter.pixelate !== 0) fx.addFilter('pixelate', config.filter.pixelate); - fx.apply(inCanvas); + outCanvas = fx.apply(inCanvas); } else { copy(inCanvas, outCanvas); // if no filters applied, output canvas is input canvas if (fx) fx = null; @@ -156,6 +156,7 @@ export function process(input: Input, config: Config, getTensor: boolean = true) } if (!getTensor) return { tensor: null, canvas: outCanvas }; // just canvas was requested + if (!outCanvas) throw new Error('cannot create output canvas'); // create tensor from image unless input was a tensor already let pixels; diff --git a/src/image/imagefx.ts b/src/image/imagefx.ts index faa05129..4374328e 100644 --- a/src/image/imagefx.ts +++ b/src/image/imagefx.ts @@ -1,15 +1,34 @@ /** * Image Filters in WebGL algoritm implementation - * * Based on: [WebGLImageFilter](https://github.com/phoboslab/WebGLImageFilter) - * * This module is written in ES5 JS and does not conform to code and style standards */ -// @ts-nocheck +import * as shaders from './imagefxshaders'; -function GLProgram(gl, vertexSource, fragmentSource) { - const _collect = function (source, prefix, collection) { +class GLProgram { + uniform = {}; + attribute = {}; + gl: WebGLRenderingContext; + id: WebGLProgram; + constructor(gl, vertexSource, fragmentSource) { + this.gl = gl; + const _vsh = this.compile(vertexSource, this.gl.VERTEX_SHADER); + const _fsh = this.compile(fragmentSource, this.gl.FRAGMENT_SHADER); + this.id = this.gl.createProgram() as WebGLProgram; + this.gl.attachShader(this.id, _vsh); + this.gl.attachShader(this.id, _fsh); + this.gl.linkProgram(this.id); + if (!this.gl.getProgramParameter(this.id, this.gl.LINK_STATUS)) throw new Error(`filter: gl link failed: ${this.gl.getProgramInfoLog(this.id)}`); + this.gl.useProgram(this.id); + this.collect(vertexSource, 'attribute', this.attribute); // Collect attributes + for (const a in this.attribute) this.attribute[a] = this.gl.getAttribLocation(this.id, a); + this.collect(vertexSource, 'uniform', this.uniform); // Collect uniforms + this.collect(fragmentSource, 'uniform', this.uniform); + for (const u in this.uniform) this.uniform[u] = this.gl.getUniformLocation(this.id, u); + } + + collect = (source, prefix, collection) => { const r = new RegExp('\\b' + prefix + ' \\w+ (\\w+)', 'ig'); source.replace(r, (match, name) => { collection[name] = 0; @@ -17,55 +36,32 @@ function GLProgram(gl, vertexSource, fragmentSource) { }); }; - const _compile = function (source, type) { - const shader = gl.createShader(type); - gl.shaderSource(shader, source); - gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error('filter: gl compile failed', gl.getShaderInfoLog(shader)); + compile = (source, type): WebGLShader => { + const shader = this.gl.createShader(type) as WebGLShader; + this.gl.shaderSource(shader, source); + this.gl.compileShader(shader); + if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) throw new Error(`filter: gl compile failed: ${this.gl.getShaderInfoLog(shader)}`); return shader; }; - - this.uniform = {}; - this.attribute = {}; - const _vsh = _compile(vertexSource, gl.VERTEX_SHADER); - const _fsh = _compile(fragmentSource, gl.FRAGMENT_SHADER); - this.id = gl.createProgram(); - gl.attachShader(this.id, _vsh); - gl.attachShader(this.id, _fsh); - gl.linkProgram(this.id); - - if (!gl.getProgramParameter(this.id, gl.LINK_STATUS)) throw new Error('filter: gl link failed', gl.getProgramInfoLog(this.id)); - - gl.useProgram(this.id); - // Collect attributes - _collect(vertexSource, 'attribute', this.attribute); - for (const a in this.attribute) this.attribute[a] = gl.getAttribLocation(this.id, a); - // Collect uniforms - _collect(vertexSource, 'uniform', this.uniform); - _collect(fragmentSource, 'uniform', this.uniform); - for (const u in this.uniform) this.uniform[u] = gl.getUniformLocation(this.id, u); } -// export const GLImageFilter = function (params) { export function GLImageFilter(params) { if (!params) params = { }; let _drawCount = 0; let _sourceTexture = null; let _lastInChain = false; let _currentFramebufferIndex = -1; - let _tempFramebuffers = [null, null]; - let _filterChain = []; + let _tempFramebuffers: [null, null] | [{ fbo: any, texture: any }] = [null, null]; + let _filterChain: Record[] = []; let _width = -1; let _height = -1; let _vertexBuffer = null; - let _currentProgram = null; - const _filter = {}; - const _canvas = params.canvas || document.createElement('canvas'); - // key is the shader program source, value is the compiled program - const _shaderProgramCache = { }; + let _currentProgram: GLProgram | null = null; + const _canvas = params.canvas || typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(100, 100) : document.createElement('canvas'); + const _shaderProgramCache = { }; // key is the shader program source, value is the compiled program const DRAW = { INTERMEDIATE: 1 }; const gl = _canvas.getContext('webgl'); - if (!gl) throw new Error('filter: context failed'); + if (!gl) throw new Error('filter: cannot get webgl context'); this.addFilter = function (name) { // eslint-disable-next-line prefer-rest-params @@ -79,27 +75,20 @@ export function GLImageFilter(params) { }; const _resize = function (width, height) { - // Same width/height? Nothing to do here - if (width === _width && height === _height) { return; } + if (width === _width && height === _height) return; // Same width/height? Nothing to do here _canvas.width = width; _width = width; _canvas.height = height; _height = height; - // Create the context if we don't have it yet - if (!_vertexBuffer) { - // Create the vertex buffer for the two triangles [x, y, u, v] * 6 - const vertices = new Float32Array([ - -1, -1, 0, 1, 1, -1, 1, 1, -1, 1, 0, 0, - -1, 1, 0, 0, 1, -1, 1, 1, 1, 1, 1, 0, - ]); + if (!_vertexBuffer) { // Create the context if we don't have it yet + const vertices = new Float32Array([-1, -1, 0, 1, 1, -1, 1, 1, -1, 1, 0, 0, -1, 1, 0, 0, 1, -1, 1, 1, 1, 1, 1, 0]); // Create the vertex buffer for the two triangles [x, y, u, v] * 6 // eslint-disable-next-line no-unused-expressions (_vertexBuffer = gl.createBuffer(), gl.bindBuffer(gl.ARRAY_BUFFER, _vertexBuffer)); gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW); gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); } gl.viewport(0, 0, _width, _height); - // Delete old temp framebuffers - _tempFramebuffers = [null, null]; + _tempFramebuffers = [null, null]; // Delete old temp framebuffers }; const _createFramebufferTexture = function (width, height) { @@ -125,58 +114,45 @@ export function GLImageFilter(params) { return _tempFramebuffers[index]; }; - const _draw = function (flags = null) { + const _draw = function (flags = 0) { + if (!_currentProgram) return; let source = null; let target = null; let flipY = false; - // Set up the source - if (_drawCount === 0) { - // First draw call - use the source texture - source = _sourceTexture; - } else { - // All following draw calls use the temp buffer last drawn to - source = _getTempFramebuffer(_currentFramebufferIndex)?.texture; - } + if (_drawCount === 0) source = _sourceTexture; // First draw call - use the source texture + else source = _getTempFramebuffer(_currentFramebufferIndex)?.texture; // All following draw calls use the temp buffer last drawn to _drawCount++; - // Set up the target - if (_lastInChain && !(flags & DRAW.INTERMEDIATE)) { - // Last filter in our chain - draw directly to the WebGL Canvas. We may - // also have to flip the image vertically now + if (_lastInChain && !(flags & DRAW.INTERMEDIATE)) { // Last filter in our chain - draw directly to the WebGL Canvas. We may also have to flip the image vertically now target = null; flipY = _drawCount % 2 === 0; } else { - // Intermediate draw call - get a temp buffer to draw to _currentFramebufferIndex = (_currentFramebufferIndex + 1) % 2; - target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo; + target = _getTempFramebuffer(_currentFramebufferIndex)?.fbo; // Intermediate draw call - get a temp buffer to draw to } - // Bind the source and target and draw the two triangles - gl.bindTexture(gl.TEXTURE_2D, source); + gl.bindTexture(gl.TEXTURE_2D, source); // Bind the source and target and draw the two triangles gl.bindFramebuffer(gl.FRAMEBUFFER, target); - gl.uniform1f(_currentProgram.uniform.flipY, (flipY ? -1 : 1)); + gl.uniform1f(_currentProgram.uniform['flipY'], (flipY ? -1 : 1)); gl.drawArrays(gl.TRIANGLES, 0, 6); }; this.apply = function (image) { _resize(image.width, image.height); _drawCount = 0; - // Create the texture for the input image if we haven't yet - if (!_sourceTexture) _sourceTexture = gl.createTexture(); + if (!_sourceTexture) _sourceTexture = gl.createTexture(); // Create the texture for the input image if we haven't yet gl.bindTexture(gl.TEXTURE_2D, _sourceTexture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); - // No filters? Just draw - if (_filterChain.length === 0) { - // const program = _compileShader(SHADER.FRAGMENT_IDENTITY); + if (_filterChain.length === 0) { // draw when done with filters _draw(); - return _canvas; - } - for (let i = 0; i < _filterChain.length; i++) { - _lastInChain = (i === _filterChain.length - 1); - const f = _filterChain[i]; - f.func.apply(this, f.args || []); + } else { // apply filters one-by-one recursively + for (let i = 0; i < _filterChain.length; i++) { + _lastInChain = (i === _filterChain.length - 1); + const f = _filterChain[i]; + f.func.apply(this, f.args || []); + } } return _canvas; }; @@ -184,355 +160,236 @@ export function GLImageFilter(params) { const _compileShader = function (fragmentSource) { if (_shaderProgramCache[fragmentSource]) { _currentProgram = _shaderProgramCache[fragmentSource]; - gl.useProgram(_currentProgram.id); + gl.useProgram(_currentProgram?.id); return _currentProgram; } - // Compile shaders - const SHADER = {}; - SHADER.VERTEX_IDENTITY = [ - 'precision highp float;', - 'attribute vec2 pos;', - 'attribute vec2 uv;', - 'varying vec2 vUv;', - 'uniform float flipY;', - 'void main(void) {', - 'vUv = uv;', - 'gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.);', - '}', - ].join('\n'); - SHADER.FRAGMENT_IDENTITY = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform sampler2D texture;', - 'void main(void) {', - 'gl_FragColor = texture2D(texture, vUv);', - '}', - ].join('\n'); - _currentProgram = new GLProgram(gl, SHADER.VERTEX_IDENTITY, fragmentSource); + _currentProgram = new GLProgram(gl, shaders.vertexIdentity, fragmentSource); const floatSize = Float32Array.BYTES_PER_ELEMENT; const vertSize = 4 * floatSize; - gl.enableVertexAttribArray(_currentProgram.attribute.pos); - gl.vertexAttribPointer(_currentProgram.attribute.pos, 2, gl.FLOAT, false, vertSize, 0 * floatSize); + gl.enableVertexAttribArray(_currentProgram.attribute['pos']); + gl.vertexAttribPointer(_currentProgram.attribute['pos'], 2, gl.FLOAT, false, vertSize, 0 * floatSize); gl.enableVertexAttribArray(_currentProgram.attribute.uv); - gl.vertexAttribPointer(_currentProgram.attribute.uv, 2, gl.FLOAT, false, vertSize, 2 * floatSize); + gl.vertexAttribPointer(_currentProgram.attribute['uv'], 2, gl.FLOAT, false, vertSize, 2 * floatSize); _shaderProgramCache[fragmentSource] = _currentProgram; return _currentProgram; }; - // ------------------------------------------------------------------------- - // Color Matrix Filter - _filter.colorMatrix = function (matrix) { - // Create a Float32 Array and normalize the offset component to 0-1 - const m = new Float32Array(matrix); - m[4] /= 255; - m[9] /= 255; - m[14] /= 255; - m[19] /= 255; - // Can we ignore the alpha value? Makes things a bit faster. - const shader = (m[18] === 1 && m[3] === 0 && m[8] === 0 && m[13] === 0 && m[15] === 0 && m[16] === 0 && m[17] === 0 && m[19] === 0) - ? _filter.colorMatrix.SHADER.WITHOUT_ALPHA - : _filter.colorMatrix.SHADER.WITH_ALPHA; - const program = _compileShader(shader); - gl.uniform1fv(program.uniform.m, m); - _draw(); + // Color Matrix Filter: Used by most color filters + const _filter = { + colorMatrix: (matrix) => { + const m = new Float32Array(matrix); // Create a Float32 Array and normalize the offset component to 0-1 + m[4] /= 255; + m[9] /= 255; + m[14] /= 255; + m[19] /= 255; + const shader = (m[18] === 1 && m[3] === 0 && m[8] === 0 && m[13] === 0 && m[15] === 0 && m[16] === 0 && m[17] === 0 && m[19] === 0) // Can we ignore the alpha value? Makes things a bit faster. + ? shaders.colorMatrixWithoutAlpha + : shaders.colorMatrixWithAlpha; + const program = _compileShader(shader); + gl.uniform1fv(program?.uniform['m'], m); + _draw(); + }, + + brightness: (brightness) => { + const b = (brightness || 0) + 1; + _filter.colorMatrix([ + b, 0, 0, 0, 0, + 0, b, 0, 0, 0, + 0, 0, b, 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + saturation: (amount) => { + const x = (amount || 0) * 2 / 3 + 1; + const y = ((x - 1) * -0.5); + _filter.colorMatrix([ + x, y, y, 0, 0, + y, x, y, 0, 0, + y, y, x, 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + desaturate: () => { + _filter.saturation(-1); + }, + + contrast: (amount) => { + const v = (amount || 0) + 1; + const o = -128 * (v - 1); + _filter.colorMatrix([ + v, 0, 0, 0, o, + 0, v, 0, 0, o, + 0, 0, v, 0, o, + 0, 0, 0, 1, 0, + ]); + }, + + negative: () => { + _filter.contrast(-2); + }, + + hue: (rotation) => { + rotation = (rotation || 0) / 180 * Math.PI; + const cos = Math.cos(rotation); + const sin = Math.sin(rotation); + const lumR = 0.213; + const lumG = 0.715; + const lumB = 0.072; + _filter.colorMatrix([ + lumR + cos * (1 - lumR) + sin * (-lumR), lumG + cos * (-lumG) + sin * (-lumG), lumB + cos * (-lumB) + sin * (1 - lumB), 0, 0, + lumR + cos * (-lumR) + sin * (0.143), lumG + cos * (1 - lumG) + sin * (0.140), lumB + cos * (-lumB) + sin * (-0.283), 0, 0, + lumR + cos * (-lumR) + sin * (-(1 - lumR)), lumG + cos * (-lumG) + sin * (lumG), lumB + cos * (1 - lumB) + sin * (lumB), 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + desaturateLuminance: () => { + _filter.colorMatrix([ + 0.2764723, 0.9297080, 0.0938197, 0, -37.1, + 0.2764723, 0.9297080, 0.0938197, 0, -37.1, + 0.2764723, 0.9297080, 0.0938197, 0, -37.1, + 0, 0, 0, 1, 0, + ]); + }, + + sepia: () => { + _filter.colorMatrix([ + 0.393, 0.7689999, 0.18899999, 0, 0, + 0.349, 0.6859999, 0.16799999, 0, 0, + 0.272, 0.5339999, 0.13099999, 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + brownie: () => { + _filter.colorMatrix([ + 0.5997023498159715, 0.34553243048391263, -0.2708298674538042, 0, 47.43192855600873, + -0.037703249837783157, 0.8609577587992641, 0.15059552388459913, 0, -36.96841498319127, + 0.24113635128153335, -0.07441037908422492, 0.44972182064877153, 0, -7.562075277591283, + 0, 0, 0, 1, 0, + ]); + }, + + vintagePinhole: () => { + _filter.colorMatrix([ + 0.6279345635605994, 0.3202183420819367, -0.03965408211312453, 0, 9.651285835294123, + 0.02578397704808868, 0.6441188644374771, 0.03259127616149294, 0, 7.462829176470591, + 0.0466055556782719, -0.0851232987247891, 0.5241648018700465, 0, 5.159190588235296, + 0, 0, 0, 1, 0, + ]); + }, + + kodachrome: () => { + _filter.colorMatrix([ + 1.1285582396593525, -0.3967382283601348, -0.03992559172921793, 0, 63.72958762196502, + -0.16404339962244616, 1.0835251566291304, -0.05498805115633132, 0, 24.732407896706203, + -0.16786010706155763, -0.5603416277695248, 1.6014850761964943, 0, 35.62982807460946, + 0, 0, 0, 1, 0, + ]); + }, + + technicolor: () => { + _filter.colorMatrix([ + 1.9125277891456083, -0.8545344976951645, -0.09155508482755585, 0, 11.793603434377337, + -0.3087833385928097, 1.7658908555458428, -0.10601743074722245, 0, -70.35205161461398, + -0.231103377548616, -0.7501899197440212, 1.847597816108189, 0, 30.950940869491138, + 0, 0, 0, 1, 0, + ]); + }, + + polaroid: () => { + _filter.colorMatrix([ + 1.438, -0.062, -0.062, 0, 0, + -0.122, 1.378, -0.122, 0, 0, + -0.016, -0.016, 1.483, 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + shiftToBGR: () => { + _filter.colorMatrix([ + 0, 0, 1, 0, 0, + 0, 1, 0, 0, 0, + 1, 0, 0, 0, 0, + 0, 0, 0, 1, 0, + ]); + }, + + // Convolution Filter + convolution: (matrix) => { + const m = new Float32Array(matrix); + const pixelSizeX = 1 / _width; + const pixelSizeY = 1 / _height; + const program = _compileShader(shaders.convolution); + gl.uniform1fv(program?.uniform['m'], m); + gl.uniform2f(program?.uniform['px'], pixelSizeX, pixelSizeY); + _draw(); + }, + + detectEdges: () => { + _filter.convolution.call(this, [ + 0, 1, 0, + 1, -4, 1, + 0, 1, 0, + ]); + }, + + sobelX: () => { + _filter.convolution.call(this, [ + -1, 0, 1, + -2, 0, 2, + -1, 0, 1, + ]); + }, + + sobelY: () => { + _filter.convolution.call(this, [ + -1, -2, -1, + 0, 0, 0, + 1, 2, 1, + ]); + }, + + sharpen: (amount) => { + const a = amount || 1; + _filter.convolution.call(this, [ + 0, -1 * a, 0, + -1 * a, 1 + 4 * a, -1 * a, + 0, -1 * a, 0, + ]); + }, + + emboss: (size) => { + const s = size || 1; + _filter.convolution.call(this, [ + -2 * s, -1 * s, 0, + -1 * s, 1, 1 * s, + 0, 1 * s, 2 * s, + ]); + }, + + // Blur Filter + blur: (size) => { + const blurSizeX = (size / 7) / _width; + const blurSizeY = (size / 7) / _height; + const program = _compileShader(shaders.blur); + // Vertical + gl.uniform2f(program?.uniform['px'], 0, blurSizeY); + _draw(DRAW.INTERMEDIATE); + // Horizontal + gl.uniform2f(program?.uniform['px'], blurSizeX, 0); + _draw(); + }, + + // Pixelate Filter + pixelate: (size) => { + const blurSizeX = (size) / _width; + const blurSizeY = (size) / _height; + const program = _compileShader(shaders.pixelate); + gl.uniform2f(program?.uniform['size'], blurSizeX, blurSizeY); + _draw(); + }, }; - _filter.colorMatrix.SHADER = {}; - _filter.colorMatrix.SHADER.WITH_ALPHA = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform sampler2D texture;', - 'uniform float m[20];', - 'void main(void) {', - 'vec4 c = texture2D(texture, vUv);', - 'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[3] * c.a + m[4];', - 'gl_FragColor.g = m[5] * c.r + m[6] * c.g + m[7] * c.b + m[8] * c.a + m[9];', - 'gl_FragColor.b = m[10] * c.r + m[11] * c.g + m[12] * c.b + m[13] * c.a + m[14];', - 'gl_FragColor.a = m[15] * c.r + m[16] * c.g + m[17] * c.b + m[18] * c.a + m[19];', - '}', - ].join('\n'); - _filter.colorMatrix.SHADER.WITHOUT_ALPHA = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform sampler2D texture;', - 'uniform float m[20];', - 'void main(void) {', - 'vec4 c = texture2D(texture, vUv);', - 'gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[4];', - 'gl_FragColor.g = m[5] * c.r + m[6] * c.g + m[7] * c.b + m[9];', - 'gl_FragColor.b = m[10] * c.r + m[11] * c.g + m[12] * c.b + m[14];', - 'gl_FragColor.a = c.a;', - '}', - ].join('\n'); - - _filter.brightness = function (brightness) { - const b = (brightness || 0) + 1; - _filter.colorMatrix([ - b, 0, 0, 0, 0, - 0, b, 0, 0, 0, - 0, 0, b, 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.saturation = function (amount) { - const x = (amount || 0) * 2 / 3 + 1; - const y = ((x - 1) * -0.5); - _filter.colorMatrix([ - x, y, y, 0, 0, - y, x, y, 0, 0, - y, y, x, 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.desaturate = function () { - _filter.saturation(-1); - }; - - _filter.contrast = function (amount) { - const v = (amount || 0) + 1; - const o = -128 * (v - 1); - - _filter.colorMatrix([ - v, 0, 0, 0, o, - 0, v, 0, 0, o, - 0, 0, v, 0, o, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.negative = function () { - _filter.contrast(-2); - }; - - _filter.hue = function (rotation) { - rotation = (rotation || 0) / 180 * Math.PI; - const cos = Math.cos(rotation); - const sin = Math.sin(rotation); - const lumR = 0.213; - const lumG = 0.715; - const lumB = 0.072; - - _filter.colorMatrix([ - lumR + cos * (1 - lumR) + sin * (-lumR), lumG + cos * (-lumG) + sin * (-lumG), lumB + cos * (-lumB) + sin * (1 - lumB), 0, 0, - lumR + cos * (-lumR) + sin * (0.143), lumG + cos * (1 - lumG) + sin * (0.140), lumB + cos * (-lumB) + sin * (-0.283), 0, 0, - lumR + cos * (-lumR) + sin * (-(1 - lumR)), lumG + cos * (-lumG) + sin * (lumG), lumB + cos * (1 - lumB) + sin * (lumB), 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.desaturateLuminance = function () { - _filter.colorMatrix([ - 0.2764723, 0.9297080, 0.0938197, 0, -37.1, - 0.2764723, 0.9297080, 0.0938197, 0, -37.1, - 0.2764723, 0.9297080, 0.0938197, 0, -37.1, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.sepia = function () { - _filter.colorMatrix([ - 0.393, 0.7689999, 0.18899999, 0, 0, - 0.349, 0.6859999, 0.16799999, 0, 0, - 0.272, 0.5339999, 0.13099999, 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.brownie = function () { - _filter.colorMatrix([ - 0.5997023498159715, 0.34553243048391263, -0.2708298674538042, 0, 47.43192855600873, - -0.037703249837783157, 0.8609577587992641, 0.15059552388459913, 0, -36.96841498319127, - 0.24113635128153335, -0.07441037908422492, 0.44972182064877153, 0, -7.562075277591283, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.vintagePinhole = function () { - _filter.colorMatrix([ - 0.6279345635605994, 0.3202183420819367, -0.03965408211312453, 0, 9.651285835294123, - 0.02578397704808868, 0.6441188644374771, 0.03259127616149294, 0, 7.462829176470591, - 0.0466055556782719, -0.0851232987247891, 0.5241648018700465, 0, 5.159190588235296, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.kodachrome = function () { - _filter.colorMatrix([ - 1.1285582396593525, -0.3967382283601348, -0.03992559172921793, 0, 63.72958762196502, - -0.16404339962244616, 1.0835251566291304, -0.05498805115633132, 0, 24.732407896706203, - -0.16786010706155763, -0.5603416277695248, 1.6014850761964943, 0, 35.62982807460946, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.technicolor = function () { - _filter.colorMatrix([ - 1.9125277891456083, -0.8545344976951645, -0.09155508482755585, 0, 11.793603434377337, - -0.3087833385928097, 1.7658908555458428, -0.10601743074722245, 0, -70.35205161461398, - -0.231103377548616, -0.7501899197440212, 1.847597816108189, 0, 30.950940869491138, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.polaroid = function () { - _filter.colorMatrix([ - 1.438, -0.062, -0.062, 0, 0, - -0.122, 1.378, -0.122, 0, 0, - -0.016, -0.016, 1.483, 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - _filter.shiftToBGR = function () { - _filter.colorMatrix([ - 0, 0, 1, 0, 0, - 0, 1, 0, 0, 0, - 1, 0, 0, 0, 0, - 0, 0, 0, 1, 0, - ]); - }; - - // ------------------------------------------------------------------------- - // Convolution Filter - _filter.convolution = function (matrix) { - const m = new Float32Array(matrix); - const pixelSizeX = 1 / _width; - const pixelSizeY = 1 / _height; - const program = _compileShader(_filter.convolution.SHADER); - gl.uniform1fv(program.uniform.m, m); - gl.uniform2f(program.uniform.px, pixelSizeX, pixelSizeY); - _draw(); - }; - - _filter.convolution.SHADER = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform sampler2D texture;', - 'uniform vec2 px;', - 'uniform float m[9];', - 'void main(void) {', - 'vec4 c11 = texture2D(texture, vUv - px);', // top left - 'vec4 c12 = texture2D(texture, vec2(vUv.x, vUv.y - px.y));', // top center - 'vec4 c13 = texture2D(texture, vec2(vUv.x + px.x, vUv.y - px.y));', // top right - 'vec4 c21 = texture2D(texture, vec2(vUv.x - px.x, vUv.y) );', // mid left - 'vec4 c22 = texture2D(texture, vUv);', // mid center - 'vec4 c23 = texture2D(texture, vec2(vUv.x + px.x, vUv.y) );', // mid right - 'vec4 c31 = texture2D(texture, vec2(vUv.x - px.x, vUv.y + px.y) );', // bottom left - 'vec4 c32 = texture2D(texture, vec2(vUv.x, vUv.y + px.y) );', // bottom center - 'vec4 c33 = texture2D(texture, vUv + px );', // bottom right - 'gl_FragColor = ', - 'c11 * m[0] + c12 * m[1] + c22 * m[2] +', - 'c21 * m[3] + c22 * m[4] + c23 * m[5] +', - 'c31 * m[6] + c32 * m[7] + c33 * m[8];', - 'gl_FragColor.a = c22.a;', - '}', - ].join('\n'); - - _filter.detectEdges = function () { - _filter.convolution.call(this, [ - 0, 1, 0, - 1, -4, 1, - 0, 1, 0, - ]); - }; - - _filter.sobelX = function () { - _filter.convolution.call(this, [ - -1, 0, 1, - -2, 0, 2, - -1, 0, 1, - ]); - }; - - _filter.sobelY = function () { - _filter.convolution.call(this, [ - -1, -2, -1, - 0, 0, 0, - 1, 2, 1, - ]); - }; - - _filter.sharpen = function (amount) { - const a = amount || 1; - _filter.convolution.call(this, [ - 0, -1 * a, 0, - -1 * a, 1 + 4 * a, -1 * a, - 0, -1 * a, 0, - ]); - }; - - _filter.emboss = function (size) { - const s = size || 1; - _filter.convolution.call(this, [ - -2 * s, -1 * s, 0, - -1 * s, 1, 1 * s, - 0, 1 * s, 2 * s, - ]); - }; - - // ------------------------------------------------------------------------- - // Blur Filter - _filter.blur = function (size) { - const blurSizeX = (size / 7) / _width; - const blurSizeY = (size / 7) / _height; - const program = _compileShader(_filter.blur.SHADER); - // Vertical - gl.uniform2f(program.uniform.px, 0, blurSizeY); - _draw(DRAW.INTERMEDIATE); - // Horizontal - gl.uniform2f(program.uniform.px, blurSizeX, 0); - _draw(); - }; - - _filter.blur.SHADER = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform sampler2D texture;', - 'uniform vec2 px;', - 'void main(void) {', - 'gl_FragColor = vec4(0.0);', - 'gl_FragColor += texture2D(texture, vUv + vec2(-7.0*px.x, -7.0*px.y))*0.0044299121055113265;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-6.0*px.x, -6.0*px.y))*0.00895781211794;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-5.0*px.x, -5.0*px.y))*0.0215963866053;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-4.0*px.x, -4.0*px.y))*0.0443683338718;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-3.0*px.x, -3.0*px.y))*0.0776744219933;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-2.0*px.x, -2.0*px.y))*0.115876621105;', - 'gl_FragColor += texture2D(texture, vUv + vec2(-1.0*px.x, -1.0*px.y))*0.147308056121;', - 'gl_FragColor += texture2D(texture, vUv )*0.159576912161;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 1.0*px.x, 1.0*px.y))*0.147308056121;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 2.0*px.x, 2.0*px.y))*0.115876621105;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 3.0*px.x, 3.0*px.y))*0.0776744219933;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 4.0*px.x, 4.0*px.y))*0.0443683338718;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 5.0*px.x, 5.0*px.y))*0.0215963866053;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 6.0*px.x, 6.0*px.y))*0.00895781211794;', - 'gl_FragColor += texture2D(texture, vUv + vec2( 7.0*px.x, 7.0*px.y))*0.0044299121055113265;', - '}', - ].join('\n'); - - // ------------------------------------------------------------------------- - // Pixelate Filter - _filter.pixelate = function (size) { - const blurSizeX = (size) / _width; - const blurSizeY = (size) / _height; - const program = _compileShader(_filter.pixelate.SHADER); - // Horizontal - gl.uniform2f(program.uniform.size, blurSizeX, blurSizeY); - _draw(); - }; - - _filter.pixelate.SHADER = [ - 'precision highp float;', - 'varying vec2 vUv;', - 'uniform vec2 size;', - 'uniform sampler2D texture;', - 'vec2 pixelate(vec2 coord, vec2 size) {', - 'return floor( coord / size ) * size;', - '}', - 'void main(void) {', - 'gl_FragColor = vec4(0.0);', - 'vec2 coord = pixelate(vUv, size);', - 'gl_FragColor += texture2D(texture, coord);', - '}', - ].join('\n'); } diff --git a/src/image/imagefxshaders.ts b/src/image/imagefxshaders.ts new file mode 100644 index 00000000..5f4a8d7a --- /dev/null +++ b/src/image/imagefxshaders.ts @@ -0,0 +1,112 @@ +export const vertexIdentity = ` + precision highp float; + attribute vec2 pos; + attribute vec2 uv; + varying vec2 vUv; + uniform float flipY; + void main(void) { + vUv = uv; + gl_Position = vec4(pos.x, pos.y*flipY, 0.0, 1.); + } +`; + +export const fragmentIdentity = ` + precision highp float; + varying vec2 vUv; + uniform sampler2D texture; + void main(void) { + gl_FragColor = texture2D(texture, vUv); + } +`; + +export const colorMatrixWithAlpha = ` + precision highp float; + varying vec2 vUv; + uniform sampler2D texture; + uniform float m[20]; + void main(void) { + vec4 c = texture2D(texture, vUv); + gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[3] * c.a + m[4]; + gl_FragColor.g = m[5] * c.r + m[6] * c.g + m[7] * c.b + m[8] * c.a + m[9]; + gl_FragColor.b = m[10] * c.r + m[11] * c.g + m[12] * c.b + m[13] * c.a + m[14]; + gl_FragColor.a = m[15] * c.r + m[16] * c.g + m[17] * c.b + m[18] * c.a + m[19]; + } +`; + +export const colorMatrixWithoutAlpha = ` + precision highp float; + varying vec2 vUv; + uniform sampler2D texture; + uniform float m[20]; + void main(void) { + vec4 c = texture2D(texture, vUv); + gl_FragColor.r = m[0] * c.r + m[1] * c.g + m[2] * c.b + m[4]; + gl_FragColor.g = m[5] * c.r + m[6] * c.g + m[7] * c.b + m[9]; + gl_FragColor.b = m[10] * c.r + m[11] * c.g + m[12] * c.b + m[14]; + gl_FragColor.a = c.a; + } +`; + +export const pixelate = ` + precision highp float; + varying vec2 vUv; + uniform vec2 size; + uniform sampler2D texture; + vec2 pixelate(vec2 coord, vec2 size) { + return floor( coord / size ) * size; + } + void main(void) { + gl_FragColor = vec4(0.0); + vec2 coord = pixelate(vUv, size); + gl_FragColor += texture2D(texture, coord); + } +`; + +export const blur = ` + precision highp float; + varying vec2 vUv; + uniform sampler2D texture; + uniform vec2 px; + void main(void) { + gl_FragColor = vec4(0.0); + gl_FragColor += texture2D(texture, vUv + vec2(-7.0*px.x, -7.0*px.y))*0.0044299121055113265; + gl_FragColor += texture2D(texture, vUv + vec2(-6.0*px.x, -6.0*px.y))*0.00895781211794; + gl_FragColor += texture2D(texture, vUv + vec2(-5.0*px.x, -5.0*px.y))*0.0215963866053; + gl_FragColor += texture2D(texture, vUv + vec2(-4.0*px.x, -4.0*px.y))*0.0443683338718; + gl_FragColor += texture2D(texture, vUv + vec2(-3.0*px.x, -3.0*px.y))*0.0776744219933; + gl_FragColor += texture2D(texture, vUv + vec2(-2.0*px.x, -2.0*px.y))*0.115876621105; + gl_FragColor += texture2D(texture, vUv + vec2(-1.0*px.x, -1.0*px.y))*0.147308056121; + gl_FragColor += texture2D(texture, vUv )*0.159576912161; + gl_FragColor += texture2D(texture, vUv + vec2( 1.0*px.x, 1.0*px.y))*0.147308056121; + gl_FragColor += texture2D(texture, vUv + vec2( 2.0*px.x, 2.0*px.y))*0.115876621105; + gl_FragColor += texture2D(texture, vUv + vec2( 3.0*px.x, 3.0*px.y))*0.0776744219933; + gl_FragColor += texture2D(texture, vUv + vec2( 4.0*px.x, 4.0*px.y))*0.0443683338718; + gl_FragColor += texture2D(texture, vUv + vec2( 5.0*px.x, 5.0*px.y))*0.0215963866053; + gl_FragColor += texture2D(texture, vUv + vec2( 6.0*px.x, 6.0*px.y))*0.00895781211794; + gl_FragColor += texture2D(texture, vUv + vec2( 7.0*px.x, 7.0*px.y))*0.0044299121055113265; + } +`; + +export const convolution = ` + precision highp float; + varying vec2 vUv; + uniform sampler2D texture; + uniform vec2 px; + uniform float m[9]; + void main(void) { + vec4 c11 = texture2D(texture, vUv - px); // top left + vec4 c12 = texture2D(texture, vec2(vUv.x, vUv.y - px.y)); // top center + vec4 c13 = texture2D(texture, vec2(vUv.x + px.x, vUv.y - px.y)); // top right + vec4 c21 = texture2D(texture, vec2(vUv.x - px.x, vUv.y) ); // mid left + vec4 c22 = texture2D(texture, vUv); // mid center + vec4 c23 = texture2D(texture, vec2(vUv.x + px.x, vUv.y) ); // mid right + vec4 c31 = texture2D(texture, vec2(vUv.x - px.x, vUv.y + px.y) ); // bottom left + vec4 c32 = texture2D(texture, vec2(vUv.x, vUv.y + px.y) ); // bottom center + vec4 c33 = texture2D(texture, vUv + px ); // bottom right + gl_FragColor = + c11 * m[0] + c12 * m[1] + c22 * m[2] + + c21 * m[3] + c22 * m[4] + c23 * m[5] + + c31 * m[6] + c32 * m[7] + c33 * m[8]; + gl_FragColor.a = c22.a; + } +`;