From 7d28cd06f2aad2a47be680bf9e58676742ece38d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 19 Sep 2021 14:07:53 -0400 Subject: [PATCH] added configuration validation --- CHANGELOG.md | 3 +- TODO.md | 14 ++++- package.json | 6 +-- src/fingerpose/estimator.ts | 1 + src/gesture/gesture.ts | 2 +- src/handpose/handdetector.ts | 3 +- src/handpose/handpipeline.ts | 3 +- src/helpers.ts | 17 +++++++ src/human.ts | 13 ++++- src/image/image.ts | 2 +- test/test-main.js | 99 +++++++++++++++++++++++++++++------- test/test-node-gpu.js | 2 +- test/test-node-wasm.js | 2 +- test/test-node.js | 12 ++--- test/test.js | 17 ++++--- wiki | 2 +- 16 files changed, 153 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcdc8cd3..6f649295 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ ## Changelog -### **HEAD -> main** 2021/09/17 mandic00@live.com +### **HEAD -> main** 2021/09/18 mandic00@live.com +- prevent validation failed on some model combinations - webgl exception handling ### **2.2.2** 2021/09/17 mandic00@live.com diff --git a/TODO.md b/TODO.md index afb9445d..b03b656e 100644 --- a/TODO.md +++ b/TODO.md @@ -30,6 +30,15 @@ Feature is automatically disabled in NodeJS without user impact
+### Face Emotion Detection + +Face Emotion detection using WASM backend has reduced precision due to math errors in backend + +- Backend WASM incorrect handling of `int32` tensors + + +
+ ### Hand Detection Enhanced rotation correction for hand detection is not working in NodeJS due to missing kernel op in TFJS @@ -38,7 +47,10 @@ Feature is automatically disabled in NodeJS without user impact - Backend NodeJS missing kernel op `RotateWithOffset` -Hand detection using WASM backend has reduced precision due to math rounding errors in backend +Hand detection using WASM backend has reduced precision due to math errors in backend + +- Backend WASM incorrect handling of `int32` tensors +
diff --git a/package.json b/package.json index 8cff1116..266c3068 100644 --- a/package.json +++ b/package.json @@ -66,11 +66,11 @@ "@tensorflow/tfjs-layers": "^3.9.0", "@tensorflow/tfjs-node": "^3.9.0", "@tensorflow/tfjs-node-gpu": "^3.9.0", - "@types/node": "^16.9.2", + "@types/node": "^16.9.3", "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", - "@vladmandic/build": "^0.5.1", - "@vladmandic/pilogger": "^0.3.2", + "@vladmandic/build": "^0.5.2", + "@vladmandic/pilogger": "^0.3.3", "canvas": "^2.8.0", "dayjs": "^1.10.7", "esbuild": "^0.12.28", diff --git a/src/fingerpose/estimator.ts b/src/fingerpose/estimator.ts index 7cee2f81..8e3275e4 100644 --- a/src/fingerpose/estimator.ts +++ b/src/fingerpose/estimator.ts @@ -21,6 +21,7 @@ function calculateSlope(point1x, point1y, point2x, point2y) { // point1, point2 are 2d or 3d point arrays (xy[z]) // returns either a single scalar (2d) or array of two slopes (3d) function getSlopes(point1, point2) { + if (!point1 || !point2) return [0, 0]; const slopeXY = calculateSlope(point1[0], point1[1], point2[0], point2[1]); if (point1.length === 2) return slopeXY; const slopeYZ = calculateSlope(point1[1], point1[2], point2[1], point2[2]); diff --git a/src/gesture/gesture.ts b/src/gesture/gesture.ts index 51489961..29779c9b 100644 --- a/src/gesture/gesture.ts +++ b/src/gesture/gesture.ts @@ -124,7 +124,7 @@ export const hand = (res): GestureResult[] => { for (let i = 0; i < res.length; i++) { const fingers: Array<{ name: string, position: number }> = []; for (const [finger, pos] of Object.entries(res[i]['annotations'])) { - if (finger !== 'palmBase' && Array.isArray(pos)) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger + if (finger !== 'palmBase' && Array.isArray(pos) && pos[0]) fingers.push({ name: finger.toLowerCase(), position: pos[0] }); // get tip of each finger } if (fingers && fingers.length > 0) { const closest = fingers.reduce((best, a) => (best.position[2] < a.position[2] ? best : a)); diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index 63ccf353..ddd83af9 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -47,7 +47,8 @@ export class HandDetector { const scores = await t.scores.data(); t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]); t.norm = this.normalizeBoxes(t.boxes); - t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 10 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); + // box detection is flaky so we look for 3x boxes than we need results + t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 3 * config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); const nms = await t.nms.array() as Array; const hands: Array<{ box: Tensor, palmLandmarks: Tensor, confidence: number }> = []; for (const index of nms) { diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index fd53a953..f869e3a7 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -22,7 +22,7 @@ export class HandPipeline { constructor(handDetector, handPoseModel) { this.handDetector = handDetector; this.handPoseModel = handPoseModel; - this.inputSize = this.handPoseModel.inputs[0].shape ? this.handPoseModel.inputs[0].shape[2] : 0; + this.inputSize = this.handPoseModel && this.handPoseModel.inputs[0].shape ? this.handPoseModel.inputs[0].shape[2] : 0; this.storedBoxes = []; this.skipped = 0; this.detectedHands = 0; @@ -152,6 +152,7 @@ export class HandPipeline { } this.storedBoxes = this.storedBoxes.filter((a) => a !== null); this.detectedHands = hands.length; + if (hands.length > config.hand.maxDetected) hands.length = config.hand.maxDetected; return hands; } } diff --git a/src/helpers.ts b/src/helpers.ts index 65d7c8f7..d591c445 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -25,6 +25,23 @@ export const now = () => { return parseInt((Number(process.hrtime.bigint()) / 1000 / 1000).toString()); }; +// helper function: checks current config validity +export function validate(defaults, config, parent = 'config', msgs: Array<{ reason: string, where: string, expected?: string }> = []) { + for (const key of Object.keys(config)) { + if (typeof config[key] === 'object') { + validate(defaults[key], config[key], key, msgs); + } else { + const defined = (typeof defaults[key] !== 'undefined'); + if (!defined) msgs.push({ reason: 'unknown property', where: `${parent}.${key} = ${config[key]}` }); + const same = typeof defaults[key] === typeof config[key]; + if (defined && !same) msgs.push({ reason: 'property type mismatch', where: `${parent}.${key} = ${config[key]}`, expected: typeof defaults[key] }); + } + // ok = ok && defined && same; + } + if (config.debug && parent === 'config' && msgs.length > 0) log('invalid configuration', msgs); + return msgs; +} + // helper function: perform deep merge of multiple objects so it allows full inheriance with overrides export function mergeDeep(...objects) { const isObject = (obj) => obj && typeof obj === 'object'; diff --git a/src/human.ts b/src/human.ts index 2e562cee..63d2e948 100644 --- a/src/human.ts +++ b/src/human.ts @@ -2,7 +2,7 @@ * Human main module */ -import { log, now, mergeDeep } from './helpers'; +import { log, now, mergeDeep, validate } from './helpers'; import { Config, defaults } from './config'; import type { Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult } from './result'; import * as tf from '../dist/tfjs.esm.js'; @@ -173,7 +173,10 @@ export class Human { defaults.backend = this.env.browser ? 'humangl' : 'tensorflow'; this.version = app.version; // expose version property on instance of class Object.defineProperty(this, 'version', { value: app.version }); // expose version property directly on class itself - this.config = mergeDeep(defaults, userConfig || {}); + this.config = JSON.parse(JSON.stringify(defaults)); + Object.seal(this.config); + if (userConfig) this.config = mergeDeep(this.config, userConfig); + validate(defaults, this.config); this.tf = tf; this.state = 'idle'; this.#numTensors = 0; @@ -248,6 +251,12 @@ export class Human { return null; } + /** Reset configuration to default values */ + reset = () => this.config = JSON.parse(JSON.stringify(defaults)); + + /** Validate current configuration schema */ + validate = (userConfig?: Partial) => validate(defaults, userConfig || this.config); + /** Process input as return canvas and tensor * * @param input: {@link Input} diff --git a/src/image/image.ts b/src/image/image.ts index 87aeccd4..428a06db 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -166,7 +166,7 @@ export function process(input: Input, config: Config): { tensor: Tensor | null, let pixels; if (outCanvas.data) { // if we have data, just convert to tensor const shape = [outCanvas.height, outCanvas.width, 3]; - pixels = tf.tensor3d(outCanvas.data, shape, 'int32'); + pixels = tf.tensor3d(outCanvas.data, shape, 'float32'); } else if ((typeof ImageData !== 'undefined') && (outCanvas instanceof ImageData)) { // if input is imagedata, just use it pixels = tf.browser ? tf.browser.fromPixels(outCanvas) : null; } else if (config.backend === 'webgl' || config.backend === 'humangl') { // tf kernel-optimized method to get imagedata diff --git a/test/test-main.js b/test/test-main.js index 8a5b4ee1..bc60c30c 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -40,14 +40,16 @@ async function getImage(human, input) { ctx.drawImage(img, 0, 0, img.width, img.height); const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const res = human.tf.tidy(() => { - const tensor = human.tf.tensor(Array.from(imageData.data), [canvas.height, canvas.width, 4], 'int32'); // create rgba image tensor from flat array + const tensor = human.tf.tensor(Array.from(imageData.data), [canvas.height, canvas.width, 4], 'float32'); // create rgba image tensor from flat array const channels = human.tf.split(tensor, 4, 2); // split rgba to channels const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb const reshape = human.tf.reshape(rgb, [1, canvas.height, canvas.width, 3]); // move extra dim from the end of tensor and use it as batch number instead return reshape; }); - if (res && res.shape[0] === 1 && res.shape[3] === 3) log('state', 'passed: load image:', input, res.shape); + const sum = human.tf.sum(res); + if (res && res.shape[0] === 1 && res.shape[3] === 3) log('state', 'passed: load image:', input, res.shape, { checksum: sum.dataSync()[0] }); else log('error', 'failed: load image:', input, res); + human.tf.dispose(sum); return res; } @@ -98,10 +100,10 @@ async function testWarmup(human, title) { // const count = human.tf.engine().state.numTensors; // if (count - tensors > 0) log('warn', 'failed: memory', config.warmup, title, 'tensors:', count - tensors); printResults(warmup); - return true; + } else { + log('error', 'failed: warmup:', config.warmup, title); } - log('error', 'failed: warmup:', config.warmup, title); - return false; + return warmup; } async function testDetect(human, input, title) { @@ -124,10 +126,10 @@ async function testDetect(human, input, title) { // const count = human.tf.engine().state.numTensors; // if (count - tensors > 0) log('warn', 'failed: memory', config.warmup, title, 'tensors:', count - tensors); printResults(detect); - return true; + } else { + log('error', 'failed: detect', input || 'random', title); } - log('error', 'failed: detect', input || 'random', title); - return false; + return detect; } const evt = { image: 0, detect: 0, warmup: 0 }; async function events(event) { @@ -144,33 +146,94 @@ async function test(Human, inputConfig) { return; } const t0 = process.hrtime.bigint(); - let human; + let res; // test event emitter - human = new Human(config); + const human = new Human(config); + human.events.addEventListener('warmup', () => events('warmup')); human.events.addEventListener('image', () => events('image')); human.events.addEventListener('detect', () => events('detect')); + // test configuration validation + let invalid = human.validate(); + if (invalid.length === 0) log('state', 'passed: configuration default validation', invalid); + else log('error', 'failed: configuration default validation', invalid); + config.invalid = true; + invalid = human.validate(config); + if (invalid.length === 1) log('state', 'passed: configuration invalid validation', invalid); + else log('error', 'failed: configuration default validation', invalid); + delete config.invalid; + // test warmup sequences await testInstance(human); config.warmup = 'none'; - await testWarmup(human, 'default'); + res = await testWarmup(human, 'default'); + if (res.error !== 'null') log('error', 'failed: warmup none result mismatch'); + else log('state', 'passed: warmup none result match'); config.warmup = 'face'; - await testWarmup(human, 'default'); + res = await testWarmup(human, 'default'); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 0 || res?.gesture?.length !== 3) log('error', 'failed: warmup face result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + else log('state', 'passed: warmup face result match'); + config.warmup = 'body'; - await testWarmup(human, 'default'); + res = await testWarmup(human, 'default'); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 0 || res?.gesture?.length !== 3) log('error', 'failed: warmup body result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + else log('state', 'passed: warmup body result match'); // test default config log('info', 'test default'); - human = new Human(config); - await testDetect(human, 'samples/ai-body.jpg', 'default'); + human.reset(); + config.cacheSensitivity = 0; + res = await testDetect(human, 'samples/ai-body.jpg', 'default'); + if (!res || res?.face?.length !== 1 || res?.face[0].gender !== 'female') log('error', 'failed: default result face mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + else log('state', 'passed: default result face match'); + + // test default config + log('info', 'test object'); + human.reset(); + config.object = { enabled: true }; + res = await testDetect(human, 'samples/ai-body.jpg', 'default'); + if (!res || res?.object?.length !== 1 || res?.object[0]?.label !== 'person') log('error', 'failed: object result mismatch', res?.object?.length); + else log('state', 'passed: object result match'); + + // test sensitive config + log('info', 'test sensitive'); + human.reset(); + config.cacheSensitivity = 0; + config.face = { detector: { minConfidence: 0.0001, maxDetected: 1 } }; + config.body = { minConfidence: 0.0001, maxDetected: 1 }; + config.hand = { minConfidence: 0.0001, maxDetected: 3 }; + res = await testDetect(human, 'samples/ai-body.jpg', 'default'); + if (!res || res?.face?.length !== 1 || res?.body?.length !== 1 || res?.hand?.length !== 3 || res?.gesture?.length !== 9) log('error', 'failed: sensitive result mismatch', res?.face?.length, res?.body?.length, res?.hand?.length, res?.gesture?.length); + else log('state', 'passed: sensitive result match'); + + // test sensitive details face + const face = res && res.face ? res.face[0] : null; + if (!face || face?.box?.length !== 4 || face?.mesh?.length !== 478 || face?.emotion?.length !== 4 || face?.embedding?.length !== 1024 || face?.rotation?.matrix?.length !== 9) { + log('error', 'failed: sensitive face result mismatch', res?.face?.length, face?.box?.length, face?.mesh?.length, face?.emotion?.length, face?.embedding?.length, face?.rotation?.matrix?.length); + } else log('state', 'passed: sensitive face result match'); + + // test sensitive details body + const body = res && res.body ? res.body[0] : null; + if (!body || body?.box?.length !== 4 || body?.keypoints?.length !== 17) log('error', 'failed: sensitive body result mismatch', body); + else log('state', 'passed: sensitive body result match'); + + // test sensitive details hand + const hand = res && res.hand ? res.hand[0] : null; + if (!hand || hand?.box?.length !== 4 || hand?.keypoints?.length !== 21) log('error', 'failed: sensitive hand result mismatch', hand?.keypoints?.length); + else log('state', 'passed: sensitive hand result match'); // test detectors only log('info', 'test detectors'); - config.face = { mesh: { enabled: false }, iris: { enabled: false }, hand: { landmarks: false } }; - human = new Human(config); - await testDetect(human, 'samples/ai-body.jpg', 'default'); + human.reset(); + config.face = { mesh: { enabled: false }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false } }; + config.hand = { landmarks: false }; + res = await testDetect(human, 'samples/ai-body.jpg', 'default'); + if (!res || res?.face?.length !== 1 || res?.face[0]?.gender || res?.face[0]?.age || res?.face[0]?.embedding) log('error', 'failed: detectors result face mismatch', res?.face); + else log('state', 'passed: detector result face match'); + if (!res || res?.hand?.length !== 1 || res?.hand[0]?.landmarks) log('error', 'failed: detectors result hand mismatch', res?.hand?.length); + else log('state', 'passed: detector result hand match'); // test posenet and movenet log('info', 'test body variants'); diff --git a/test/test-node-gpu.js b/test/test-node-gpu.js index 6c5e4892..2ea9bb47 100644 --- a/test/test-node-gpu.js +++ b/test/test-node-gpu.js @@ -8,7 +8,7 @@ const config = { async: true, face: { enabled: true, - detector: { enabled: true, rotation: true }, + detector: { rotation: true }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, diff --git a/test/test-node-wasm.js b/test/test-node-wasm.js index fcbb5da7..6c63614f 100644 --- a/test/test-node-wasm.js +++ b/test/test-node-wasm.js @@ -19,7 +19,7 @@ const config = { async: false, face: { enabled: true, - detector: { enabled: true, rotation: false }, + detector: { rotation: false }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, diff --git a/test/test-node.js b/test/test-node.js index 6186226e..9253e20d 100644 --- a/test/test-node.js +++ b/test/test-node.js @@ -5,19 +5,19 @@ const config = { modelBasePath: 'file://models/', backend: 'tensorflow', debug: false, - async: false, + async: true, face: { enabled: true, - detector: { enabled: true, rotation: true }, + detector: { rotation: true }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, }, - hand: { enabled: false, rotation: true }, - body: { enabled: false }, - object: { enabled: false }, - segmentation: { enabled: false }, + hand: { enabled: true }, + body: { enabled: true }, + object: { enabled: true }, + segmentation: { enabled: true }, filter: { enabled: false }, }; diff --git a/test/test.js b/test/test.js index ae0798bd..c885d087 100644 --- a/test/test.js +++ b/test/test.js @@ -23,15 +23,18 @@ const ignoreMessages = [ 'ExperimentalWarning', ]; -const status = { - passed: 0, - failed: 0, -}; +const status = {}; function logMessage(test, data) { - log[data[0]](test, ...data[1]); - if (data[1][0].startsWith('passed')) status.passed++; - if (data[1][0].startsWith('failed')) status.failed++; + if (!status[test]) status[test] = { passed: 0, failed: 0 }; + if (log[data[0]]) { + log[data[0]](test, ...data[1]); + } else { + log.error('unknown facility', test, ...data[1]); + status[test].failed++; + } + if (data[1][0].startsWith('passed')) status[test].passed++; + if (data[1][0].startsWith('failed')) status[test].failed++; } function logStdIO(ok, test, buffer) { diff --git a/wiki b/wiki index 740fcd1b..b24eafa2 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 740fcd1b5b2cc92bde0111b630e872cbf7670c81 +Subproject commit b24eafa265bda331788e0d36cf5c854a494e33d6