ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix rotation interpolation

pull/356/head
Vladimir Mandic 2022-09-21 13:51:49 -04:00
parent b636eedc6b
commit 4d8369bff2
4 changed files with 60 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human # @vladmandic/human
Version: **2.10.2** Version: **2.10.3**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>** Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,9 +9,13 @@
## Changelog ## Changelog
### **HEAD -> main** 2022/09/17 mandic00@live.com ### **HEAD -> main** 2022/09/21 mandic00@live.com
### **origin/main** 2022/09/19 mandic00@live.com
- add human.video method
### **2.10.2** 2022/09/11 mandic00@live.com ### **2.10.2** 2022/09/11 mandic00@live.com
- add node.js esm compatibility (#292) - add node.js esm compatibility (#292)

1
TODO.md
View File

@ -47,6 +47,7 @@ Enable via `about:config` -> `gfx.offscreencanvas.enabled`
Runs continous detection of an input video instead of processing each frame manually Runs continous detection of an input video instead of processing each frame manually
- New simple demo [*Live*](https://vladmandic.github.io/human/demo/video/index.html) - New simple demo [*Live*](https://vladmandic.github.io/human/demo/video/index.html)
- Enable model cache when using web workers - Enable model cache when using web workers
- Fix for `face.rotation` interpolation
- Improve NodeJS resolver when using ESM - Improve NodeJS resolver when using ESM
- Update demo `demo/nodejs/process-folder.js` - Update demo `demo/nodejs/process-folder.js`
and re-process `/samples` and re-process `/samples`

25
src/util/interpolate.ts
View File

@ -22,13 +22,15 @@ export function calc(newResult: Result, config: Config): Result {
// thus mixing by-reference and by-value assignments to minimize memory operations // thus mixing by-reference and by-value assignments to minimize memory operations
const elapsed = Date.now() - newResult.timestamp; const elapsed = Date.now() - newResult.timestamp;
// curve fitted: buffer = 8 - ln(delay)
// interpolation formula: current = ((buffer - 1) * previous + live) / buffer /* curve fitted: buffer = 8 - ln(delay)
// - at 50ms delay buffer = ~4.1 => 28% towards live data interpolation formula: current = ((buffer - 1) * previous + live) / buffer
// - at 250ms delay buffer = ~2.5 => 40% towards live data - at 50ms delay buffer = ~4.1 => 28% towards live data
// - at 500ms delay buffer = ~1.8 => 55% towards live data - at 250ms delay buffer = ~2.5 => 40% towards live data
// - at 750ms delay buffer = ~1.4 => 71% towards live data - at 500ms delay buffer = ~1.8 => 55% towards live data
// - at 1sec delay buffer = 1 which means live data is used - at 750ms delay buffer = ~1.4 => 71% towards live data
- at 1sec delay buffer = 1 which means live data is used
*/
const bufferedFactor = elapsed < 1000 ? 8 - Math.log(elapsed + 1) : 1; const bufferedFactor = elapsed < 1000 ? 8 - Math.log(elapsed + 1) : 1;
if (newResult.canvas) bufferedResult.canvas = newResult.canvas; if (newResult.canvas) bufferedResult.canvas = newResult.canvas;
@ -131,9 +133,9 @@ export function calc(newResult: Result, config: Config): Result {
} = { matrix: [0, 0, 0, 0, 0, 0, 0, 0, 0], angle: { roll: 0, yaw: 0, pitch: 0 }, gaze: { bearing: 0, strength: 0 } }; } = { matrix: [0, 0, 0, 0, 0, 0, 0, 0, 0], angle: { roll: 0, yaw: 0, pitch: 0 }, gaze: { bearing: 0, strength: 0 } };
rotation.matrix = newResult.face[i].rotation?.matrix as [number, number, number, number, number, number, number, number, number]; rotation.matrix = newResult.face[i].rotation?.matrix as [number, number, number, number, number, number, number, number, number];
rotation.angle = { rotation.angle = {
roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.roll || 0) + (newResult.face[i].rotation?.angle.roll || 0)) / bufferedFactor, roll: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.roll || 0) + (newResult.face[i].rotation?.angle?.roll || 0)) / bufferedFactor,
yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.yaw || 0) + (newResult.face[i].rotation?.angle.yaw || 0)) / bufferedFactor, yaw: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.yaw || 0) + (newResult.face[i].rotation?.angle?.yaw || 0)) / bufferedFactor,
pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle.pitch || 0) + (newResult.face[i].rotation?.angle.pitch || 0)) / bufferedFactor, pitch: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.angle?.pitch || 0) + (newResult.face[i].rotation?.angle?.pitch || 0)) / bufferedFactor,
}; };
rotation.gaze = { rotation.gaze = {
// not fully correct due projection on circle, also causes wrap-around draw on jump from negative to positive // not fully correct due projection on circle, also causes wrap-around draw on jump from negative to positive
@ -141,8 +143,9 @@ export function calc(newResult: Result, config: Config): Result {
strength: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze.strength || 0) + (newResult.face[i].rotation?.gaze.strength || 0)) / bufferedFactor, strength: ((bufferedFactor - 1) * (bufferedResult.face[i].rotation?.gaze.strength || 0) + (newResult.face[i].rotation?.gaze.strength || 0)) / bufferedFactor,
}; };
bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values bufferedResult.face[i] = { ...newResult.face[i], rotation, box, boxRaw }; // shallow clone plus updated values
} else {
bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
} }
bufferedResult.face[i] = { ...newResult.face[i], box, boxRaw }; // shallow clone plus updated values
} }
} }

78
test/build.log
View File

@ -1,39 +1,39 @@
2022-09-19 10:44:57 DATA:  Build {"name":"@vladmandic/human","version":"2.10.2"} 2022-09-21 13:49:15 DATA:  Build {"name":"@vladmandic/human","version":"2.10.3"}
2022-09-19 10:44:57 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.2"} 2022-09-21 13:49:15 INFO:  Application: {"name":"@vladmandic/human","version":"2.10.3"}
2022-09-19 10:44:57 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} 2022-09-21 13:49:15 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-09-19 10:44:57 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.8","typescript":"4.8.3","typedoc":"0.23.15","eslint":"8.23.1"} 2022-09-21 13:49:15 INFO:  Toolchain: {"build":"0.7.12","esbuild":"0.15.8","typescript":"4.8.3","typedoc":"0.23.15","eslint":"8.23.1"}
2022-09-19 10:44:57 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} 2022-09-21 13:49:15 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-09-19 10:44:57 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} 2022-09-21 13:49:15 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":658512,"outputBytes":309346} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":75,"inputBytes":658536,"outputBytes":309564}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":658516,"outputBytes":309350} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":75,"inputBytes":658540,"outputBytes":309568}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":658568,"outputBytes":309400} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":75,"inputBytes":658592,"outputBytes":309618}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":658487,"outputBytes":308219} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":75,"inputBytes":658511,"outputBytes":308423}
2022-09-19 10:44:57 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914} 2022-09-21 13:49:15 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1344,"outputBytes":2821914}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3479818,"outputBytes":1688397} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":75,"inputBytes":3479842,"outputBytes":1688553}
2022-09-19 10:44:57 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3479818,"outputBytes":3110170} 2022-09-21 13:49:15 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":75,"inputBytes":3479842,"outputBytes":3110385}
2022-09-19 10:45:02 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} 2022-09-21 13:49:20 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15}
2022-09-19 10:45:04 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true} 2022-09-21 13:49:22 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":77,"generated":true}
2022-09-19 10:45:04 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134} 2022-09-21 13:49:22 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6714,"outputBytes":3134}
2022-09-19 10:45:04 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788} 2022-09-21 13:49:22 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15488,"outputBytes":7788}
2022-09-19 10:45:15 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0} 2022-09-21 13:49:33 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":110,"errors":0,"warnings":0}
2022-09-19 10:45:15 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} 2022-09-21 13:49:33 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-09-19 10:45:15 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"} 2022-09-21 13:49:33 STATE: Copy: {"input":"tfjs/tfjs.esm.d.ts"}
2022-09-19 10:45:15 INFO:  Done... 2022-09-21 13:49:33 INFO:  Done...
2022-09-19 10:45:16 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193} 2022-09-21 13:49:34 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":193}
2022-09-19 10:45:16 STATE: Copy: {"input":"types/human.d.ts"} 2022-09-21 13:49:34 STATE: Copy: {"input":"types/human.d.ts"}
2022-09-19 10:45:16 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} 2022-09-21 13:49:34 INFO:  Analyze models: {"folders":8,"result":"models/models.json"}
2022-09-19 10:45:16 STATE: Models {"folder":"./models","models":13} 2022-09-21 13:49:34 STATE: Models {"folder":"./models","models":13}
2022-09-19 10:45:16 STATE: Models {"folder":"../human-models/models","models":42} 2022-09-21 13:49:34 STATE: Models {"folder":"../human-models/models","models":42}
2022-09-19 10:45:16 STATE: Models {"folder":"../blazepose/model/","models":4} 2022-09-21 13:49:34 STATE: Models {"folder":"../blazepose/model/","models":4}
2022-09-19 10:45:16 STATE: Models {"folder":"../anti-spoofing/model","models":1} 2022-09-21 13:49:34 STATE: Models {"folder":"../anti-spoofing/model","models":1}
2022-09-19 10:45:16 STATE: Models {"folder":"../efficientpose/models","models":3} 2022-09-21 13:49:34 STATE: Models {"folder":"../efficientpose/models","models":3}
2022-09-19 10:45:16 STATE: Models {"folder":"../insightface/models","models":5} 2022-09-21 13:49:34 STATE: Models {"folder":"../insightface/models","models":5}
2022-09-19 10:45:16 STATE: Models {"folder":"../movenet/models","models":3} 2022-09-21 13:49:34 STATE: Models {"folder":"../movenet/models","models":3}
2022-09-19 10:45:16 STATE: Models {"folder":"../nanodet/models","models":4} 2022-09-21 13:49:34 STATE: Models {"folder":"../nanodet/models","models":4}
2022-09-19 10:45:17 STATE: Models: {"count":57,"totalSize":383017442} 2022-09-21 13:49:34 STATE: Models: {"count":57,"totalSize":383017442}
2022-09-19 10:45:17 INFO:  Human Build complete... {"logFile":"test/build.log"} 2022-09-21 13:49:34 INFO:  Human Build complete... {"logFile":"test/build.log"}