From 4d8369bff22f6c2d6d7b507bc01413580160ea5b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 21 Sep 2022 13:51:49 -0400 Subject: [PATCH] fix rotation interpolation --- CHANGELOG.md | 8 +++-- TODO.md | 1 + src/util/interpolate.ts | 25 +++++++------ test/build.log | 78 ++++++++++++++++++++--------------------- 4 files changed, 60 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1be2a0cb..3bc8e070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human - Version: **2.10.2** + Version: **2.10.3** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,9 +9,13 @@ ## Changelog -### **HEAD -> main** 2022/09/17 mandic00@live.com +### **HEAD -> main** 2022/09/21 mandic00@live.com +### **origin/main** 2022/09/19 mandic00@live.com + +- add human.video method + ### **2.10.2** 2022/09/11 mandic00@live.com - add node.js esm compatibility (#292) diff --git a/TODO.md b/TODO.md index bccda86c..ce426494 100644 --- a/TODO.md +++ b/TODO.md @@ -47,6 +47,7 @@ Enable via `about:config` -> `gfx.offscreencanvas.enabled` Runs continous detection of an input video instead of processing each frame manually - New simple demo [*Live*](https://vladmandic.github.io/human/demo/video/index.html) - Enable model cache when using web workers +- Fix for `face.rotation` interpolation - Improve NodeJS resolver when using ESM - Update demo `demo/nodejs/process-folder.js` and re-process `/samples` diff --git a/src/util/interpolate.ts b/src/util/interpolate.ts index 4d2f2268..ce94df4d 100644 --- a/src/util/interpolate.ts +++ b/src/util/interpolate.ts @@ -22,13 +22,15 @@ export function calc(newResult: Result, config: Config): Result { // thus mixing by-reference and by-value assignments to minimize memory operations const elapsed = Date.now() - newResult.timestamp; - // curve fitted: buffer = 8 - ln(delay) - // interpolation formula: current = ((buffer - 1) * previous + live) / buffer - // - at 50ms delay buffer = ~4.1 => 28% towards live data - // - at 250ms delay buffer = ~2.5 => 40% towards live data - // - at 500ms delay buffer = ~1.8 => 55% towards live data - // - at 750ms delay buffer = ~1.4 => 71% towards live data - // - at 1sec delay buffer = 1 which means live data is used + + /* curve fitted: buffer = 8 - ln(delay) + interpolation formula: current = ((buffer - 1) * previous + live) / buffer + - at 50ms delay buffer = ~4.1 => 28% towards live data + - at 250ms delay buffer = ~2.5 => 40% towards live data + - at 500ms delay buffer = ~1.8 => 55% towards live data + - at 750ms delay buffer = ~1.4 => 71% towards live data + - at 1sec delay buffer = 1 which means live data is used + */ const bufferedFactor = elapsed < 1000 ? 8 - Math.log(elapsed + 1) : 1; if (newResult.canvas) bufferedResult.canvas = newResult.canvas; @@ -131,9 +133,9 @@ export function calc(newResult: Result, config: Config): Result { } = { matrix: [0, 0, 0, 0, 0, 0, 0, 0, 0], angle: { roll: 0, yaw: 0, pitch: 0 }, gaze: { bearing: 0, strength: 0 } }; rotation.matrix = newResult.face[i].rotation?.matrix as [number, number, number, number, number, number, number, number, number]; rotation.angle = { - roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.roll || 0) + (newResult.face[i].rotation?.angle.roll || 0)) / bufferedFactor, - yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.yaw || 0) + (newResult.face[i].rotation?.angle.yaw || 0)) / bufferedFactor, - pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.pitch || 0) + (newResult.face[i].rotation?.angle.pitch || 0)) / bufferedFactor, + roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.roll || 0) + (newResult.face[i].rotation?.angle?.roll || 0)) / bufferedFactor, + yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.yaw || 0) + (newResult.face[i].rotation?.angle?.yaw || 0)) / bufferedFactor, + pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.pitch || 0) + (newResult.face[i].rotation?.angle?.pitch || 0)) / bufferedFactor, }; rotation.gaze = { // not fully correct due projection on circle, also causes wrap-around draw on jump from negative to positive @@ -141,8 +143,9 @@ export function calc(newResult: Result, config: Config): Result { strength: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze.strength || 0) + (newResult.face[i].rotation?.gaze.strength || 0)) / bufferedFactor, }; bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values + } else { + bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values } - bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values } } diff --git a/test/build.log b/test/build.log index 5dbafc30..ad64d4e7 100644 --- a/test/build.log +++ b/test/build.log @@ -1,39 +1,39 @@ -2022-09-19 10:44:57 DATA:  Build {"name":"@vladmandic/human","version":"2.10.2"} -2022-09-19 10:44:57 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.2"} -2022-09-19 10:44:57 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} -2022-09-19 10:44:57 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.8","typescript":"4.8.3","typedoc":"0.23.15","eslint":"8.23.1"} -2022-09-19 10:44:57 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} -2022-09-19 10:44:57 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":658512,"outputBytes":309346} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":658516,"outputBytes":309350} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":658568,"outputBytes":309400} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":658487,"outputBytes":308219} -2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3479818,"outputBytes":1688397} -2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3479818,"outputBytes":3110170} -2022-09-19 10:45:02 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} -2022-09-19 10:45:04 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true} -2022-09-19 10:45:04 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134} -2022-09-19 10:45:04 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788} -2022-09-19 10:45:15 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0} -2022-09-19 10:45:15 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} -2022-09-19 10:45:15 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"} -2022-09-19 10:45:15 INFO:  Done... -2022-09-19 10:45:16 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193} -2022-09-19 10:45:16 STATE: Copy: {"input":"types/human.d.ts"} -2022-09-19 10:45:16 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} -2022-09-19 10:45:16 STATE: Models {"folder":"./models","models":13} -2022-09-19 10:45:16 STATE: Models {"folder":"../human-models/models","models":42} -2022-09-19 10:45:16 STATE: Models {"folder":"../blazepose/model/","models":4} -2022-09-19 10:45:16 STATE: Models {"folder":"../anti-spoofing/model","models":1} -2022-09-19 10:45:16 STATE: Models {"folder":"../efficientpose/models","models":3} -2022-09-19 10:45:16 STATE: Models {"folder":"../insightface/models","models":5} -2022-09-19 10:45:16 STATE: Models {"folder":"../movenet/models","models":3} -2022-09-19 10:45:16 STATE: Models {"folder":"../nanodet/models","models":4} -2022-09-19 10:45:17 STATE: Models: {"count":57,"totalSize":383017442} -2022-09-19 10:45:17 INFO:  Human Build complete... {"logFile":"test/build.log"} +2022-09-21 13:49:15 DATA:  Build {"name":"@vladmandic/human","version":"2.10.3"} +2022-09-21 13:49:15 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.3"} +2022-09-21 13:49:15 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} +2022-09-21 13:49:15 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.8","typescript":"4.8.3","typedoc":"0.23.15","eslint":"8.23.1"} +2022-09-21 13:49:15 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} +2022-09-21 13:49:15 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":658536,"outputBytes":309564} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":658540,"outputBytes":309568} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":658592,"outputBytes":309618} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":658511,"outputBytes":308423} +2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3479842,"outputBytes":1688553} +2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3479842,"outputBytes":3110385} +2022-09-21 13:49:20 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} +2022-09-21 13:49:22 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true} +2022-09-21 13:49:22 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134} +2022-09-21 13:49:22 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788} +2022-09-21 13:49:33 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0} +2022-09-21 13:49:33 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} +2022-09-21 13:49:33 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"} +2022-09-21 13:49:33 INFO:  Done... +2022-09-21 13:49:34 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193} +2022-09-21 13:49:34 STATE: Copy: {"input":"types/human.d.ts"} +2022-09-21 13:49:34 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} +2022-09-21 13:49:34 STATE: Models {"folder":"./models","models":13} +2022-09-21 13:49:34 STATE: Models {"folder":"../human-models/models","models":42} +2022-09-21 13:49:34 STATE: Models {"folder":"../blazepose/model/","models":4} +2022-09-21 13:49:34 STATE: Models {"folder":"../anti-spoofing/model","models":1} +2022-09-21 13:49:34 STATE: Models {"folder":"../efficientpose/models","models":3} +2022-09-21 13:49:34 STATE: Models {"folder":"../insightface/models","models":5} +2022-09-21 13:49:34 STATE: Models {"folder":"../movenet/models","models":3} +2022-09-21 13:49:34 STATE: Models {"folder":"../nanodet/models","models":4} +2022-09-21 13:49:34 STATE: Models: {"count":57,"totalSize":383017442} +2022-09-21 13:49:34 INFO:  Human Build complete... {"logFile":"test/build.log"}