From 12fc9079dc35f20325cca71224cc8b919be3b50c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 16 Nov 2021 20:16:49 -0500 Subject: [PATCH] optimize centernet --- CHANGELOG.md | 4 +++- demo/nodejs/node-canvas.js | 17 ++++++++++------- src/body/blazepose.ts | 2 +- src/body/efficientpose.ts | 2 +- src/face/blazeface.ts | 2 +- src/face/facemeshutil.ts | 2 +- src/face/faceres.ts | 2 +- src/gear/emotion.ts | 2 +- src/gear/ssrnet-age.ts | 2 +- src/gear/ssrnet-gender.ts | 2 +- src/hand/handposedetector.ts | 2 +- src/hand/handposepipeline.ts | 2 +- src/hand/handtrack.ts | 2 +- src/object/centernet.ts | 29 ++++++++++++----------------- src/object/nanodet.ts | 2 +- src/segmentation/segmentation.ts | 2 +- src/tfjs/backend.ts | 2 ++ src/tfjs/constants.ts | 23 +++++++++++++++++------ 18 files changed, 57 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee3d0d3..98c8e263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ ## Changelog -### **release 2.5.2** 2021/11/15 mandic00@live.com +### **HEAD -> main** 2021/11/16 mandic00@live.com +- add extra face rotation prior to mesh +- release 2.5.2 - improve error handling ### **2.5.2** 2021/11/14 mandic00@live.com diff --git a/demo/nodejs/node-canvas.js b/demo/nodejs/node-canvas.js index 8637cfc2..fd0b20f2 100644 --- a/demo/nodejs/node-canvas.js +++ b/demo/nodejs/node-canvas.js @@ -44,12 +44,12 @@ async function main() { const inputImage = await canvas.loadImage(input); // load image using canvas library log.info('Loaded image', input, inputImage.width, inputImage.height); const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas - const ctx = inputCanvas.getContext('2d'); - ctx.drawImage(inputImage, 0, 0); // draw input image onto canvas + const inputCtx = inputCanvas.getContext('2d'); + inputCtx.drawImage(inputImage, 0, 0); // draw input image onto canvas + const imageData = inputCtx.getImageData(0, 0, inputCanvas.width, inputCanvas.height); // run detection - const result = await human.detect(inputCanvas); - + const result = await human.detect(imageData); // run segmentation // const seg = await human.segmentation(inputCanvas); // log.data('Segmentation:', { data: seg.data.length, alpha: typeof seg.alpha, canvas: typeof seg.canvas }); @@ -65,11 +65,14 @@ async function main() { } // draw detected results onto canvas and save it to a file - human.draw.all(inputCanvas, result); // use human build-in method to draw results as overlays on canvas + const outputCanvas = new canvas.Canvas(inputImage.width, inputImage.height); // create canvas + const outputCtx = outputCanvas.getContext('2d'); + outputCtx.drawImage(result.canvas || inputImage, 0, 0); // draw input image onto canvas + human.draw.all(outputCanvas, result); // use human build-in method to draw results as overlays on canvas const outFile = fs.createWriteStream(output); // write canvas to new image file - outFile.on('finish', () => log.state('Output image:', output, inputCanvas.width, inputCanvas.height)); + outFile.on('finish', () => log.state('Output image:', output, outputCanvas.width, outputCanvas.height)); outFile.on('error', (err) => log.error('Output error:', output, err)); - const stream = inputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true }); + const stream = outputCanvas.createJPEGStream({ quality: 0.5, progressive: true, chromaSubsampling: true }); stream.pipe(outFile); } } diff --git a/src/body/blazepose.ts b/src/body/blazepose.ts index 0f229817..0e0da2c5 100644 --- a/src/body/blazepose.ts +++ b/src/body/blazepose.ts @@ -3,7 +3,7 @@ */ import * as tf from '../../dist/tfjs.esm.js'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import { log, join, now } from '../util/util'; import type { BodyKeypoint, BodyResult, Box, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; diff --git a/src/body/efficientpose.ts b/src/body/efficientpose.ts index b3504a36..b70e0d0f 100644 --- a/src/body/efficientpose.ts +++ b/src/body/efficientpose.ts @@ -7,7 +7,7 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import * as coords from './efficientposecoords'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { BodyResult, Point } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; diff --git a/src/face/blazeface.ts b/src/face/blazeface.ts index b234e60f..501d22f1 100644 --- a/src/face/blazeface.ts +++ b/src/face/blazeface.ts @@ -6,7 +6,7 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import * as util from './facemeshutil'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Config } from '../config'; import type { Tensor, GraphModel } from '../tfjs/types'; import { env } from '../util/env'; diff --git a/src/face/facemeshutil.ts b/src/face/facemeshutil.ts index de17ab1f..a97e9cb8 100644 --- a/src/face/facemeshutil.ts +++ b/src/face/facemeshutil.ts @@ -5,7 +5,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as coords from './facemeshcoords'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Box, Point } from '../result'; import { env } from '../util/env'; diff --git a/src/face/faceres.ts b/src/face/faceres.ts index da93cc48..2bf655b2 100644 --- a/src/face/faceres.ts +++ b/src/face/faceres.ts @@ -10,7 +10,7 @@ import { log, join, now } from '../util/util'; import { env } from '../util/env'; import * as tf from '../../dist/tfjs.esm.js'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Tensor, GraphModel } from '../tfjs/types'; import type { Config } from '../config'; diff --git a/src/gear/emotion.ts b/src/gear/emotion.ts index 37574b1a..3eb5dc0f 100644 --- a/src/gear/emotion.ts +++ b/src/gear/emotion.ts @@ -9,7 +9,7 @@ import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; import * as tf from '../../dist/tfjs.esm.js'; import { env } from '../util/env'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; const annotations = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']; let model: GraphModel | null; diff --git a/src/gear/ssrnet-age.ts b/src/gear/ssrnet-age.ts index b07cd409..c9b9da2a 100644 --- a/src/gear/ssrnet-age.ts +++ b/src/gear/ssrnet-age.ts @@ -7,7 +7,7 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import { env } from '../util/env'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; diff --git a/src/gear/ssrnet-gender.ts b/src/gear/ssrnet-gender.ts index c579a029..1fa10c0b 100644 --- a/src/gear/ssrnet-gender.ts +++ b/src/gear/ssrnet-gender.ts @@ -6,7 +6,7 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Config } from '../config'; import type { GraphModel, Tensor } from '../tfjs/types'; import { env } from '../util/env'; diff --git a/src/hand/handposedetector.ts b/src/hand/handposedetector.ts index 484daffe..81a11597 100644 --- a/src/hand/handposedetector.ts +++ b/src/hand/handposedetector.ts @@ -6,7 +6,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as util from './handposeutil'; import * as anchors from './handposeanchors'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Tensor, GraphModel } from '../tfjs/types'; import type { Point } from '../result'; diff --git a/src/hand/handposepipeline.ts b/src/hand/handposepipeline.ts index a7e3ee0f..cfa0030d 100644 --- a/src/hand/handposepipeline.ts +++ b/src/hand/handposepipeline.ts @@ -6,7 +6,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import * as util from './handposeutil'; import type * as detector from './handposedetector'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { Tensor, GraphModel } from '../tfjs/types'; import { env } from '../util/env'; import { now } from '../util/util'; diff --git a/src/hand/handtrack.ts b/src/hand/handtrack.ts index ab1c6049..b3491714 100644 --- a/src/hand/handtrack.ts +++ b/src/hand/handtrack.ts @@ -15,7 +15,7 @@ import type { Config } from '../config'; import { env } from '../util/env'; import * as fingerPose from './fingerpose'; import { fakeOps } from '../tfjs/backend'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; const models: [GraphModel | null, GraphModel | null] = [null, null]; const modelOutputNodes = ['StatefulPartitionedCall/Postprocessor/Slice', 'StatefulPartitionedCall/Postprocessor/ExpandDims_1']; diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 1f36ec31..cafd031a 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -33,26 +33,20 @@ export async function load(config: Config): Promise { async function process(res: Tensor | null, outputShape, config: Config) { if (!res) return []; + const t: Record = {}; const results: Array = []; const detections = await res.array(); - const squeezeT = tf.squeeze(res); - tf.dispose(res); - const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class - tf.dispose(squeezeT); - const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x - const boxesT = tf.squeeze(stackT); - tf.dispose(stackT); - const scoresT = tf.squeeze(arr[4]); - const classesT = tf.squeeze(arr[5]); - arr.forEach((t) => tf.dispose(t)); - const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); - tf.dispose(boxesT); - tf.dispose(scoresT); - tf.dispose(classesT); - const nms = await nmsT.data(); - tf.dispose(nmsT); + t.squeeze = tf.squeeze(res); + const arr = tf.split(t.squeeze, 6, 1) as Tensor[]; // x1, y1, x2, y2, score, class + t.stack = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x + t.boxes = tf.squeeze(t.stack); + t.scores = tf.squeeze(arr[4]); + t.classes = tf.squeeze(arr[5]); + tf.dispose([res, ...arr]); + t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); + const nms = await t.nms.data(); let i = 0; - for (const id of nms) { + for (const id of Array.from(nms)) { const score = Math.trunc(100 * detections[0][id][4]) / 100; const classVal = detections[0][id][5]; const label = labels[classVal].label; @@ -74,6 +68,7 @@ async function process(res: Tensor | null, outputShape, config: Config) { ]; results.push({ id: i++, score, class: classVal, label, box, boxRaw }); } + Object.keys(t).forEach((tensor) => tf.dispose(t[tensor])); return results; } diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index adca2414..e8d9f3e8 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -6,7 +6,7 @@ import { log, join, now } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import { labels } from './labels'; import type { ObjectResult, Box } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; diff --git a/src/segmentation/segmentation.ts b/src/segmentation/segmentation.ts index 2ca7d17c..33a4c999 100644 --- a/src/segmentation/segmentation.ts +++ b/src/segmentation/segmentation.ts @@ -9,7 +9,7 @@ import { log, join } from '../util/util'; import * as tf from '../../dist/tfjs.esm.js'; import * as image from '../image/image'; -import * as constants from '../tfjs/constants'; +import { constants } from '../tfjs/constants'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; import { env } from '../util/env'; diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index de3a27f1..57dede6a 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -5,6 +5,7 @@ import { log, now } from '../util/util'; import { env } from '../util/env'; import * as humangl from './humangl'; import * as tf from '../../dist/tfjs.esm.js'; +import * as constants from './constants'; function registerCustomOps() { if (!env.kernels.includes('mod')) { @@ -87,6 +88,7 @@ export async function check(instance: Human, force = false) { try { await tf.setBackend(instance.config.backend); await tf.ready(); + constants.init(); } catch (err) { log('error: cannot set backend:', instance.config.backend, err); return false; diff --git a/src/tfjs/constants.ts b/src/tfjs/constants.ts index c42f28f1..334f0135 100644 --- a/src/tfjs/constants.ts +++ b/src/tfjs/constants.ts @@ -1,9 +1,20 @@ import * as tf from '../../dist/tfjs.esm.js'; import type { Tensor } from './types'; -export const tf255: Tensor = tf.scalar(255, 'float32'); -export const tf1: Tensor = tf.scalar(1, 'float32'); -export const tf2: Tensor = tf.scalar(2, 'float32'); -export const tf05: Tensor = tf.scalar(0.5, 'float32'); -export const tf127: Tensor = tf.scalar(127.5, 'float32'); -export const rgb: Tensor = tf.tensor1d([0.2989, 0.5870, 0.1140], 'float32'); // factors for red/green/blue colors when converting to grayscale +export const constants: Record = { + tf255: 255, + tf1: 1, + tf2: 2, + tf05: 0.5, + tf127: 127.5, + rgb: [0.2989, 0.5870, 0.1140], +}; + +export function init() { + constants.tf255 = tf.scalar(255, 'float32'); + constants.tf1 = tf.scalar(1, 'float32'); + constants.tf2 = tf.scalar(2, 'float32'); + constants.tf05 = tf.scalar(0.5, 'float32'); + constants.tf127 = tf.scalar(127.5, 'float32'); + constants.rgb = tf.tensor1d([0.2989, 0.5870, 0.1140], 'float32'); // factors for red/green/blue colors when converting to grayscale +}