add histogram equalization

TODO.md

@@ -2,17 +2,15 @@
 
 ## Work in Progress
 
-- Switch to custom `tfjs` for main `human` ESM bundle
-- Sort out incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
-
 <br>
 
 ### Exploring
 
 - Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
-- Histogram Equalization: Regular, Adaptive, Contrast Limited
+- Histogram Equalization: Regular, Adaptive, Contrast Limited, CLAHE
 - TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
 - Body segmentation: `robust-video-matting`
+- TFJS incompatibility with latest `long.js` 5.0.0 due to CJS to ESM switch
 
 <br><hr><br>
 

demo/typescript/index.html

@@ -26,5 +26,6 @@
     <pre id="status" style="position: absolute; top: 12px; right: 20px; background-color: grey; padding: 8px; box-shadow: 2px 2px black"></pre>
     <pre id="log" style="padding: 8px"></pre>
     <div id="performance" style="position: absolute; bottom: 0; width: 100%; padding: 8px; font-size: 0.8rem;"></div>
+    <canvas id="test" style="position: absolute; bottom: 0; right: 0; width: 30%"></canvas>
   </body>
 </html>

dist/human.esm-nobundle.js

@@ -119,6 +119,7 @@ var config = {
   skipAllowed: false,
   filter: {
     enabled: true,
+    equalization: false,
     width: 0,
     height: 0,
     flip: false,
@@ -930,6 +931,21 @@ function GLImageFilter() {
   };
 }
 
+// src/image/enhance.ts
+function histogramEqualization(input) {
+  const channels = tfjs_esm_exports.split(input, 3, 2);
+  const min2 = [tfjs_esm_exports.min(channels[0]), tfjs_esm_exports.min(channels[1]), tfjs_esm_exports.min(channels[2])];
+  const max4 = [tfjs_esm_exports.max(channels[0]), tfjs_esm_exports.max(channels[1]), tfjs_esm_exports.max(channels[2])];
+  const sub6 = [tfjs_esm_exports.sub(channels[0], min2[0]), tfjs_esm_exports.sub(channels[1], min2[1]), tfjs_esm_exports.sub(channels[2], min2[2])];
+  const range = [tfjs_esm_exports.sub(max4[0], min2[0]), tfjs_esm_exports.sub(max4[1], min2[1]), tfjs_esm_exports.sub(max4[2], min2[2])];
+  const fact = [tfjs_esm_exports.div(255, range[0]), tfjs_esm_exports.div(255, range[1]), tfjs_esm_exports.div(255, range[2])];
+  const enh = [tfjs_esm_exports.mul(sub6[0], fact[0]), tfjs_esm_exports.mul(sub6[1], fact[1]), tfjs_esm_exports.mul(sub6[2], fact[2])];
+  const rgb2 = tfjs_esm_exports.stack([enh[0], enh[1], enh[2]], 2);
+  const reshape8 = tfjs_esm_exports.reshape(rgb2, [1, input.shape[0], input.shape[1], 3]);
+  tfjs_esm_exports.dispose([...channels, ...min2, ...max4, ...sub6, ...range, ...fact, ...enh, rgb2]);
+  return reshape8;
+}
+
 // src/image/image.ts
 var maxSize = 2048;
 var inCanvas = null;
@@ -1115,7 +1131,7 @@ function process2(input, config3, getTensor = true) {
     if (!pixels)
       throw new Error("cannot create tensor from input");
     const casted = tfjs_esm_exports.cast(pixels, "float32");
-    const tensor3 = tfjs_esm_exports.expandDims(casted, 0);
+    const tensor3 = config3.filter.equalization ? histogramEqualization(casted) : tfjs_esm_exports.expandDims(casted, 0);
     tfjs_esm_exports.dispose([pixels, casted]);
     return { tensor: tensor3, canvas: config3.filter.return ? outCanvas : null };
   }
@@ -5304,8 +5320,8 @@ async function predict4(image25, config3) {
       if (!(model5 == null ? void 0 : model5.inputs[0].shape))
         return null;
       const resize = tfjs_esm_exports.image.resizeBilinear(image25, [model5.inputs[0].shape[2], model5.inputs[0].shape[1]], false);
-      const enhance2 = tfjs_esm_exports.mul(resize, 2);
+      const enhance3 = tfjs_esm_exports.mul(resize, 2);
-      const norm = enhance2.sub(1);
+      const norm = enhance3.sub(1);
       return norm;
     });
     let resT;
@@ -5317,10 +5333,10 @@ async function predict4(image25, config3) {
       cache2.keypoints.length = 0;
       const squeeze8 = resT.squeeze();
       tfjs_esm_exports.dispose(resT);
-      const stack3 = squeeze8.unstack(2);
+      const stack4 = squeeze8.unstack(2);
       tfjs_esm_exports.dispose(squeeze8);
-      for (let id = 0; id < stack3.length; id++) {
+      for (let id = 0; id < stack4.length; id++) {
-        const [x2, y2, partScore] = max2d(stack3[id], config3.body.minConfidence);
+        const [x2, y2, partScore] = max2d(stack4[id], config3.body.minConfidence);
         if (partScore > (((_a = config3.body) == null ? void 0 : _a.minConfidence) || 0)) {
           cache2.keypoints.push({
             score: Math.round(100 * partScore) / 100,
@@ -5336,7 +5352,7 @@ async function predict4(image25, config3) {
           });
         }
       }
-      stack3.forEach((s) => tfjs_esm_exports.dispose(s));
+      stack4.forEach((s) => tfjs_esm_exports.dispose(s));
     }
     cache2.score = cache2.keypoints.reduce((prev, curr) => curr.score > prev ? curr.score : prev, 0);
     const x = cache2.keypoints.map((a) => a.position[0]);
@@ -5701,7 +5717,7 @@ async function load9(config3) {
     log("cached model:", modelUrl);
   return model9;
 }
-function enhance(input) {
+function enhance2(input) {
   const tensor3 = input.image || input.tensor || input;
   if (!(model9 == null ? void 0 : model9.inputs[0].shape))
     return tensor3;
@@ -5730,7 +5746,7 @@ async function predict7(image25, config3, idx, count2) {
       descriptor: []
     };
     if ((_a2 = config3.face.description) == null ? void 0 : _a2.enabled) {
-      const enhanced = enhance(image25);
+      const enhanced = enhance2(image25);
       const resT = model9 == null ? void 0 : model9.execute(enhanced);
       lastTime7 = now();
       tfjs_esm_exports.dispose(enhanced);
@@ -9524,18 +9540,18 @@ async function load10(config3) {
 // src/util/box.ts
 function calc(keypoints, outputSize2 = [1, 1]) {
   const coords8 = [keypoints.map((pt) => pt[0]), keypoints.map((pt) => pt[1])];
-  const min = [Math.min(...coords8[0]), Math.min(...coords8[1])];
+  const min2 = [Math.min(...coords8[0]), Math.min(...coords8[1])];
-  const max3 = [Math.max(...coords8[0]), Math.max(...coords8[1])];
+  const max4 = [Math.max(...coords8[0]), Math.max(...coords8[1])];
-  const box4 = [min[0], min[1], max3[0] - min[0], max3[1] - min[1]];
+  const box4 = [min2[0], min2[1], max4[0] - min2[0], max4[1] - min2[1]];
   const boxRaw = [box4[0] / outputSize2[0], box4[1] / outputSize2[1], box4[2] / outputSize2[0], box4[3] / outputSize2[1]];
   return { box: box4, boxRaw };
 }
 function square(keypoints, outputSize2 = [1, 1]) {
   const coords8 = [keypoints.map((pt) => pt[0]), keypoints.map((pt) => pt[1])];
-  const min = [Math.min(...coords8[0]), Math.min(...coords8[1])];
+  const min2 = [Math.min(...coords8[0]), Math.min(...coords8[1])];
-  const max3 = [Math.max(...coords8[0]), Math.max(...coords8[1])];
+  const max4 = [Math.max(...coords8[0]), Math.max(...coords8[1])];
-  const center = [(min[0] + max3[0]) / 2, (min[1] + max3[1]) / 2];
+  const center = [(min2[0] + max4[0]) / 2, (min2[1] + max4[1]) / 2];
-  const dist = Math.max(center[0] - min[0], center[1] - min[1], -center[0] + max3[0], -center[1] + max3[1]);
+  const dist = Math.max(center[0] - min2[0], center[1] - min2[1], -center[0] + max4[0], -center[1] + max4[1]);
   const box4 = [Math.trunc(center[0] - dist), Math.trunc(center[1] - dist), Math.trunc(2 * dist), Math.trunc(2 * dist)];
   const boxRaw = [box4[0] / outputSize2[0], box4[1] / outputSize2[1], box4[2] / outputSize2[0], box4[3] / outputSize2[1]];
   return { box: box4, boxRaw };
@@ -10253,11 +10269,11 @@ var MaxHeap = class {
     this.swim(this.numberOfElements);
   }
   dequeue() {
-    const max3 = this.priorityQueue[0];
+    const max4 = this.priorityQueue[0];
     this.exchange(0, this.numberOfElements--);
     this.sink(0);
     this.priorityQueue[this.numberOfElements + 1] = null;
-    return max3;
+    return max4;
   }
   empty() {
     return this.numberOfElements === -1;
@@ -10314,11 +10330,11 @@ function getImageCoords(part, outputStride2, offsets) {
     y: part.heatmapY * outputStride2 + y
   };
 }
-function clamp(a, min, max3) {
+function clamp(a, min2, max4) {
-  if (a < min)
+  if (a < min2)
-    return min;
+    return min2;
-  if (a > max3)
+  if (a > max4)
-    return max3;
+    return max4;
   return a;
 }
 function squaredDistance(y1, x1, y2, x2) {
@@ -12845,7 +12861,7 @@ var Human = class {
     return process5(input, background, this.config);
   }
   enhance(input) {
-    return enhance(input);
+    return enhance2(input);
   }
   async init() {
     await check(this, true);

dist/human.esm.js

@@ -108,6 +108,7 @@ var config = {
   skipAllowed: false,
   filter: {
     enabled: true,
+    equalization: false,
     width: 0,
     height: 0,
     flip: false,
@@ -71257,6 +71258,21 @@ function GLImageFilter() {
   };
 }
 
+// src/image/enhance.ts
+function histogramEqualization(input2) {
+  const channels = split(input2, 3, 2);
+  const min7 = [min(channels[0]), min(channels[1]), min(channels[2])];
+  const max7 = [max(channels[0]), max(channels[1]), max(channels[2])];
+  const sub5 = [sub(channels[0], min7[0]), sub(channels[1], min7[1]), sub(channels[2], min7[2])];
+  const range7 = [sub(max7[0], min7[0]), sub(max7[1], min7[1]), sub(max7[2], min7[2])];
+  const fact = [div(255, range7[0]), div(255, range7[1]), div(255, range7[2])];
+  const enh = [mul(sub5[0], fact[0]), mul(sub5[1], fact[1]), mul(sub5[2], fact[2])];
+  const rgb2 = stack([enh[0], enh[1], enh[2]], 2);
+  const reshape7 = reshape(rgb2, [1, input2.shape[0], input2.shape[1], 3]);
+  dispose([...channels, ...min7, ...max7, ...sub5, ...range7, ...fact, ...enh, rgb2]);
+  return reshape7;
+}
+
 // src/image/image.ts
 var maxSize = 2048;
 var inCanvas = null;
@@ -71442,7 +71458,7 @@ function process2(input2, config3, getTensor2 = true) {
     if (!pixels)
       throw new Error("cannot create tensor from input");
     const casted = cast(pixels, "float32");
-    const tensor2 = expandDims(casted, 0);
+    const tensor2 = config3.filter.equalization ? histogramEqualization(casted) : expandDims(casted, 0);
     dispose([pixels, casted]);
     return { tensor: tensor2, canvas: config3.filter.return ? outCanvas : null };
   }
@@ -75631,8 +75647,8 @@ async function predict4(image7, config3) {
       if (!(model6 == null ? void 0 : model6.inputs[0].shape))
         return null;
       const resize = image.resizeBilinear(image7, [model6.inputs[0].shape[2], model6.inputs[0].shape[1]], false);
-      const enhance2 = mul(resize, 2);
+      const enhance3 = mul(resize, 2);
-      const norm2 = enhance2.sub(1);
+      const norm2 = enhance3.sub(1);
       return norm2;
     });
     let resT;
@@ -76028,7 +76044,7 @@ async function load9(config3) {
     log("cached model:", modelUrl);
   return model10;
 }
-function enhance(input2) {
+function enhance2(input2) {
   const tensor2 = input2.image || input2.tensor || input2;
   if (!(model10 == null ? void 0 : model10.inputs[0].shape))
     return tensor2;
@@ -76057,7 +76073,7 @@ async function predict7(image7, config3, idx, count3) {
       descriptor: []
     };
     if ((_a2 = config3.face.description) == null ? void 0 : _a2.enabled) {
-      const enhanced = enhance(image7);
+      const enhanced = enhance2(image7);
       const resT = model10 == null ? void 0 : model10.execute(enhanced);
       lastTime7 = now();
       dispose(enhanced);
@@ -83172,7 +83188,7 @@ var Human = class {
     return process5(input2, background, this.config);
   }
   enhance(input2) {
-    return enhance(input2);
+    return enhance2(input2);
   }
   async init() {
     await check(this, true);

src/config.ts

@@ -122,16 +122,20 @@ export interface SegmentationConfig extends GenericConfig {
 export interface FilterConfig {
   /** @property are image filters enabled? */
   enabled: boolean,
-  /** Resize input width
+  /** @property perform image histogram equalization */
+  equalization: boolean,
+  /** resize input width
   * - if both width and height are set to 0, there is no resizing
   * - if just one is set, second one is scaled automatically
   * - if both are set, values are used as-is
+  * @property
   */
   width: number,
-  /** Resize input height
+  /** resize input height
   * - if both width and height are set to 0, there is no resizing
   * - if just one is set, second one is scaled automatically
   * - if both are set, values are used as-is
+  * @property
   */
   height: number,
   /** @property return processed canvas imagedata in result */
@@ -262,6 +266,7 @@ const config: Config = {
   skipAllowed: false,
   filter: {
     enabled: true,
+    equalization: false,
     width: 0,
     height: 0,
     flip: false,

src/image/enhance.ts

@@ -0,0 +1,20 @@
+/**
+ * Image enhancements
+ */
+
+import * as tf from '../../dist/tfjs.esm.js';
+import type { Tensor } from '../exports';
+
+export function histogramEqualization(input: Tensor): Tensor {
+  const channels = tf.split(input, 3, 2);
+  const min: Tensor[] = [tf.min(channels[0]), tf.min(channels[1]), tf.min(channels[2])];
+  const max: Tensor[] = [tf.max(channels[0]), tf.max(channels[1]), tf.max(channels[2])];
+  const sub = [tf.sub(channels[0], min[0]), tf.sub(channels[1], min[1]), tf.sub(channels[2], min[2])];
+  const range = [tf.sub(max[0], min[0]), tf.sub(max[1], min[1]), tf.sub(max[2], min[2])];
+  const fact = [tf.div(255, range[0]), tf.div(255, range[1]), tf.div(255, range[2])];
+  const enh = [tf.mul(sub[0], fact[0]), tf.mul(sub[1], fact[1]), tf.mul(sub[2], fact[2])];
+  const rgb = tf.stack([enh[0], enh[1], enh[2]], 2);
+  const reshape = tf.reshape(rgb, [1, input.shape[0], input.shape[1], 3]);
+  tf.dispose([...channels, ...min, ...max, ...sub, ...range, ...fact, ...enh, rgb]);
+  return reshape;
+}

src/image/image.ts

@@ -7,6 +7,7 @@ import * as fxImage from './imagefx';
 import type { Input, AnyCanvas, Tensor, Config } from '../exports';
 import { env } from '../util/env';
 import { log, now } from '../util/util';
+import * as enhance from './enhance';
 
 const maxSize = 2048;
 // internal temp canvases
@@ -201,7 +202,7 @@ export function process(input: Input, config: Config, getTensor: boolean = true)
     }
     if (!pixels) throw new Error('cannot create tensor from input');
     const casted = tf.cast(pixels, 'float32');
-    const tensor = tf.expandDims(casted, 0);
+    const tensor = config.filter.equalization ? enhance.histogramEqualization(casted) : tf.expandDims(casted, 0);
     tf.dispose([pixels, casted]);
     return { tensor, canvas: (config.filter.return ? outCanvas : null) };
   }

wiki

@@ -1 +1 @@
-Subproject commit 5e89af1004860ea9f302e516699b5e0b4e0a825f
+Subproject commit 6abe315e2ae3e3aa457c4c728d9e2959d7e023db