optimize centernet

CHANGELOG.md

@@ -9,8 +9,10 @@
   
 ## Changelog
   
-### **release 2.5.2** 2021/11/15 mandic00@live.com
+### **HEAD -> main** 2021/11/16 mandic00@live.com
 
+- add extra face rotation prior to mesh
+- release 2.5.2
 - improve error handling
 
 ### **2.5.2** 2021/11/14 mandic00@live.com

demo/nodejs/node-canvas.js

@@ -44,12 +44,12 @@ async function main() {
     const inputImage = await canvas.loadImage(input); // load image using canvas library
     log.info('Loaded image', input, inputImage.width, inputImage.height);
     const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas
-    const ctx = inputCanvas.getContext('2d');
-    ctx.drawImage(inputImage, 0, 0); // draw input image onto canvas
+    const inputCtx = inputCanvas.getContext('2d');
+    inputCtx.drawImage(inputImage, 0, 0); // draw input image onto canvas
+    const imageData = inputCtx.getImageData(0, 0, inputCanvas.width, inputCanvas.height);
 
     // run detection
-    const result = await human.detect(inputCanvas);
-
+    const result = await human.detect(imageData);
     // run segmentation
     // const seg = await human.segmentation(inputCanvas);
     // log.data('Segmentation:', { data: seg.data.length, alpha: typeof seg.alpha, canvas: typeof seg.canvas });
@@ -65,11 +65,14 @@ async function main() {
     }
 
     // draw detected results onto canvas and save it to a file
-    human.draw.all(inputCanvas, result); // use human build-in method to draw results as overlays on canvas
+    const outputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas
+    const outputCtx = outputCanvas.getContext('2d');
+    outputCtx.drawImage(result.canvas || inputImage, 0, 0); // draw input image onto canvas
+    human.draw.all(outputCanvas, result); // use human build-in method to draw results as overlays on canvas
     const outFile = fs.createWriteStream(output); // write canvas to new image file
-    outFile.on('finish', () => log.state('Output image:', output, inputCanvas.width, inputCanvas.height));
+    outFile.on('finish', () => log.state('Output image:', output, outputCanvas.width, outputCanvas.height));
     outFile.on('error', (err) => log.error('Output error:', output, err));
-    const stream = inputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
+    const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true });
     stream.pipe(outFile);
   }
 }

src/body/blazepose.ts

@@ -3,7 +3,7 @@
  */
 
 import * as tf from '../../dist/tfjs.esm.js';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import { log, join, now } from '../util/util';
 import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';

src/body/efficientpose.ts

@@ -7,7 +7,7 @@
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './efficientposecoords';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { BodyResult, Point } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import type { Config } from '../config';

src/face/blazeface.ts

@@ -6,7 +6,7 @@
 import { log, join } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as util from './facemeshutil';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Config } from '../config';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import { env } from '../util/env';

src/face/facemeshutil.ts

@@ -5,7 +5,7 @@
 
 import * as tf from '../../dist/tfjs.esm.js';
 import * as coords from './facemeshcoords';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Box, Point } from '../result';
 import { env } from '../util/env';
 

src/face/faceres.ts

@@ -10,7 +10,7 @@
 import { log, join, now } from '../util/util';
 import { env } from '../util/env';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import type { Config } from '../config';
 

src/gear/emotion.ts

@@ -9,7 +9,7 @@ import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import * as tf from '../../dist/tfjs.esm.js';
 import { env } from '../util/env';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 
 const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'];
 let model: GraphModel | null;

src/gear/ssrnet-age.ts

@@ -7,7 +7,7 @@
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
 import { env } from '../util/env';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 

src/gear/ssrnet-gender.ts

@@ -6,7 +6,7 @@
 
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Config } from '../config';
 import type { GraphModel, Tensor } from '../tfjs/types';
 import { env } from '../util/env';

src/hand/handposedetector.ts

@@ -6,7 +6,7 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import * as util from './handposeutil';
 import * as anchors from './handposeanchors';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import type { Point } from '../result';
 

src/hand/handposepipeline.ts

@@ -6,7 +6,7 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import * as util from './handposeutil';
 import type * as detector from './handposedetector';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import type { Tensor, GraphModel } from '../tfjs/types';
 import { env } from '../util/env';
 import { now } from '../util/util';

src/hand/handtrack.ts

@@ -15,7 +15,7 @@ import type { Config } from '../config';
 import { env } from '../util/env';
 import * as fingerPose from './fingerpose';
 import { fakeOps } from '../tfjs/backend';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 
 const models: [GraphModel | null, GraphModel | null] = [null, null];
 const modelOutputNodes = ['StatefulPartitionedCall/Postprocessor/Slice', 'StatefulPartitionedCall/Postprocessor/ExpandDims_1'];

src/object/centernet.ts

@@ -33,26 +33,20 @@ export async function load(config: Config): Promise<GraphModel> {
 
 async function process(res: Tensor | null, outputShape, config: Config) {
   if (!res) return [];
+  const t: Record<string, Tensor> = {};
   const results: Array<ObjectResult> = [];
   const detections = await res.array();
-  const squeezeT = tf.squeeze(res);
-  tf.dispose(res);
-  const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class
-  tf.dispose(squeezeT);
-  const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x
-  const boxesT = tf.squeeze(stackT);
-  tf.dispose(stackT);
-  const scoresT = tf.squeeze(arr[4]);
-  const classesT = tf.squeeze(arr[5]);
-  arr.forEach((t) => tf.dispose(t));
-  const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
-  tf.dispose(boxesT);
-  tf.dispose(scoresT);
-  tf.dispose(classesT);
-  const nms = await nmsT.data();
-  tf.dispose(nmsT);
+  t.squeeze = tf.squeeze(res);
+  const arr = tf.split(t.squeeze, 6, 1) as Tensor[]; // x1, y1, x2, y2, score, class
+  t.stack = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x
+  t.boxes = tf.squeeze(t.stack);
+  t.scores = tf.squeeze(arr[4]);
+  t.classes = tf.squeeze(arr[5]);
+  tf.dispose([res, ...arr]);
+  t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
+  const nms = await t.nms.data();
   let i = 0;
-  for (const id of nms) {
+  for (const id of Array.from(nms)) {
     const score = Math.trunc(100 * detections[0][id][4]) / 100;
     const classVal = detections[0][id][5];
     const label = labels[classVal].label;
@@ -74,6 +68,7 @@ async function process(res: Tensor | null, outputShape, config: Config) {
     ];
     results.push({ id: i++, score, class: classVal, label, box, boxRaw });
   }
+  Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
   return results;
 }
 

src/object/nanodet.ts

@@ -6,7 +6,7 @@
 
 import { log, join, now } from '../util/util';
 import * as tf from '../../dist/tfjs.esm.js';
-import * as constants from '../tfjs/constants';
+import { constants } from '../tfjs/constants';
 import { labels } from './labels';
 import type { ObjectResult, Box } from '../result';
 import type { GraphModel, Tensor } from '../tfjs/types';

src/tfjs/backend.ts

@@ -5,6 +5,7 @@ import { log, now } from '../util/util';
 import { env } from '../util/env';
 import * as humangl from './humangl';
 import * as tf from '../../dist/tfjs.esm.js';
+import * as constants from './constants';
 
 function registerCustomOps() {
   if (!env.kernels.includes('mod')) {
@@ -87,6 +88,7 @@ export async function check(instance: Human, force = false) {
       try {
         await tf.setBackend(instance.config.backend);
         await tf.ready();
+        constants.init();
       } catch (err) {
         log('error: cannot set backend:', instance.config.backend, err);
         return false;

src/tfjs/constants.ts

@@ -1,9 +1,20 @@
 import * as tf from '../../dist/tfjs.esm.js';
 import type { Tensor } from './types';
 
-export const tf255: Tensor = tf.scalar(255, 'float32');
-export const tf1: Tensor = tf.scalar(1, 'float32');
-export const tf2: Tensor = tf.scalar(2, 'float32');
-export const tf05: Tensor = tf.scalar(0.5, 'float32');
-export const tf127: Tensor = tf.scalar(127.5, 'float32');
-export const rgb: Tensor = tf.tensor1d([0.2989, 0.5870, 0.1140], 'float32'); // factors for red/green/blue colors when converting to grayscale
+export const constants: Record<string, Tensor | number | number[]> = {
+  tf255: 255,
+  tf1: 1,
+  tf2: 2,
+  tf05: 0.5,
+  tf127: 127.5,
+  rgb: [0.2989, 0.5870, 0.1140],
+};
+
+export function init() {
+  constants.tf255 = tf.scalar(255, 'float32');
+  constants.tf1 = tf.scalar(1, 'float32');
+  constants.tf2 = tf.scalar(2, 'float32');
+  constants.tf05 = tf.scalar(0.5, 'float32');
+  constants.tf127 = tf.scalar(127.5, 'float32');
+  constants.rgb = tf.tensor1d([0.2989, 0.5870, 0.1140], 'float32'); // factors for red/green/blue colors when converting to grayscale
+}