From cc71f647bf8aedd240165010b41c95bb5c82ddbb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 24 Sep 2021 09:55:27 -0400 Subject: [PATCH] new release --- CHANGELOG.md | 6 +++++- TODO.md | 3 --- src/config.ts | 32 +++++++++++++++++++++++++++----- src/handtrack/handtrack.ts | 24 +----------------------- src/human.ts | 11 ++++++++--- test/node.js | 8 +++++++- test/test-main.js | 19 +++++++++++++++++-- 7 files changed, 65 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 870fde98..f07df2ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,12 @@ ## Changelog -### **HEAD -> main** 2021/09/22 mandic00@live.com +### **2.2.3** 2021/09/24 mandic00@live.com + +### **origin/main** 2021/09/23 mandic00@live.com + +- support segmentation for nodejs - redo segmentation and handtracking - prototype handtracking - automated browser tests diff --git a/TODO.md b/TODO.md index 2f65c791..f13fb64b 100644 --- a/TODO.md +++ b/TODO.md @@ -6,9 +6,6 @@ ### Handtrack -- Finish implementation -- Set defaults and image sizes -- Optimize model - Add tests
diff --git a/src/config.ts b/src/config.ts index c8de876a..25e4e313 100644 --- a/src/config.ts +++ b/src/config.ts @@ -211,13 +211,26 @@ export interface GestureConfig { * * Contains all configurable parameters * @typedef Config + * + * Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L292) */ export interface Config { - /** Backend used for TFJS operations */ + /** Backend used for TFJS operations + * Valid build-in backends are: + * - Browser: `cpu`, `wasm`, `webgl`, `humangl` + * - NodeJS: `cpu`, `wasm`, `tensorflow` + * + * Experimental: + * - Browser: `webgpu` - requires custom build of `tfjs-backend-webgpu` + * + * Defaults: `humangl` for browser and `tensorflow` for nodejs + */ backend: '' | 'cpu' | 'wasm' | 'webgl' | 'humangl' | 'tensorflow' | 'webgpu', // backend: string; - /** Path to *.wasm files if backend is set to `wasm` */ + /** Path to *.wasm files if backend is set to `wasm` + * - if not set, auto-detects to link to CDN `jsdelivr` when running in browser + */ wasmPath: string, /** Print debug statements to console */ @@ -248,20 +261,27 @@ export interface Config { /** Run input through image filters before inference * - image filters run with near-zero latency as they are executed on the GPU + * + * {@link FilterConfig} */ filter: Partial, - // type definition end + /** {@link GestureConfig} */ gesture: Partial; + /** {@link FaceConfig} */ face: Partial, + /** {@link BodyConfig} */ body: Partial, + /** {@link HandConfig} */ hand: Partial, + /** {@link ObjectConfig} */ object: Partial, + /** {@link SegmentationConfig} */ segmentation: Partial, } @@ -332,7 +352,7 @@ const config: Config = { skipFrames: 15, // how many max frames to go without re-running the face bounding box detector // only used when cacheSensitivity is not zero // e.g., if model is running st 25 FPS, we can re-use existing bounding - // box for updated face analysis as the head probably hasn't moved much + // box for updated face analysis as the head does not move fast // in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.2, // threshold for discarding a prediction iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed @@ -386,10 +406,11 @@ const config: Config = { enabled: true, rotation: true, // use best-guess rotated hand image or just box with rotation as-is // false means higher performance, but incorrect finger mapping if hand is inverted + // only valid for `handdetect` variation skipFrames: 18, // how many max frames to go without re-running the hand bounding box detector // only used when cacheSensitivity is not zero // e.g., if model is running st 25 FPS, we can re-use existing bounding - // box for updated hand skeleton analysis as the hand probably + // box for updated hand skeleton analysis as the hand // hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.8, // threshold for discarding a prediction iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed @@ -427,4 +448,5 @@ const config: Config = { blur: 8, // blur segmentation output by n pixels for more realistic image }, }; + export { config as defaults }; diff --git a/src/handtrack/handtrack.ts b/src/handtrack/handtrack.ts index e847dad3..555cab23 100644 --- a/src/handtrack/handtrack.ts +++ b/src/handtrack/handtrack.ts @@ -58,6 +58,7 @@ const fingerMap = { }; export async function loadDetect(config: Config): Promise { + // HandTrack Model: Original: TFJS Port: if (env.initial) models[0] = null; if (!models[0]) { // handtrack model has some kernel ops defined in model but those are never referenced and non-existent in tfjs @@ -224,26 +225,3 @@ export async function predict(input: Tensor, config: Config): Promise -- TFJS Port: -- Original: -- Writeup: -- Convert: - tensorflowjs_converter --input_format=tf_frozen_model --output_format=tfjs_graph_model \ - --output_node_names='num_detections,detection_boxes,detection_scores,detection_classes' --saved_model_tags=serve --quantize_uint8=* \ - --strip_debug_ops=* --weight_shard_size_bytes=10000000000 --control_flow_v2=true frozen_inference_graph.pb graph - -webmodel/efficientdet512d0/base/model.json -webmodel/centernet512fpn/base/model.json -https://github.com/victordibia/handtrack.js/commit/70d5d9c98e69688414cddaad044bd8730bc982d1#diff-c40e819be4ec1dc29f26913f5cdeb05202261b3a1725ab259cb235ea0f0fc5d6 - -git rev-list HEAD -- webmodel/* - 9ba7220fb31e9168aa248500cc70800566f4c719 - 70d5d9c98e69688414cddaad044bd8730bc982d1 - -git checkout 9ba7220fb31e9168aa248500cc70800566f4c719^ -- webmodel -git checkout 70d5d9c98e69688414cddaad044bd8730bc982d1^ -- webmodel - -*/ diff --git a/src/human.ts b/src/human.ts index c9abc6f1..5a663f0e 100644 --- a/src/human.ts +++ b/src/human.ts @@ -80,13 +80,15 @@ export class Human { version: string; /** Current configuration - * - Details: {@link Config} + * - Definition: {@link Config} + * - Defaults: [config](https://github.com/vladmandic/human/blob/main/src/config.ts#L292) */ config: Config; /** Last known result of detect run * - Can be accessed anytime after initial detection - */ + * - Definition: {@link Result} + */ result: Result; /** Current state of Human library @@ -101,6 +103,8 @@ export class Human { /** Instance of TensorFlow/JS used by Human * - Can be embedded or externally provided * @internal + * + * [TFJS API] */ tf: TensorFlow; @@ -119,6 +123,7 @@ export class Human { /** Currently loaded models * @internal + * {@link Models} */ models: models.Models; @@ -150,7 +155,7 @@ export class Human { * * @param userConfig: {@link Config} * - * @return instance + * @return instance: {@link Human} */ constructor(userConfig?: Partial) { env.get(); diff --git a/test/node.js b/test/node.js index 1fb4f3d5..adeeb9dd 100644 --- a/test/node.js +++ b/test/node.js @@ -31,6 +31,8 @@ const ignoreMessages = [ 'ExperimentalWarning', ]; +const failedMessages = []; + const status = {}; function logMessage(test, data) { @@ -42,7 +44,10 @@ function logMessage(test, data) { status[test].failed++; } if (data[1][0].startsWith('passed')) status[test].passed++; - if (data[1][0].startsWith('failed')) status[test].failed++; + if (data[1][0].startsWith('failed')) { + status[test].failed++; + failedMessages.push({ test, data }); + } } function logStdIO(ok, test, buffer) { @@ -100,6 +105,7 @@ async function testAll() { // for (const demo of demos) await runDemo(demo); for (const test of tests) await runTest(test); log.info(); + log.info('failed', failedMessages); log.info('status:', status); } diff --git a/test/test-main.js b/test/test-main.js index 62e93925..1c8cd97c 100644 --- a/test/test-main.js +++ b/test/test-main.js @@ -302,9 +302,24 @@ async function test(Human, inputConfig) { // test posenet and movenet log('info', 'test body variants'); config.body = { modelPath: 'posenet.json' }; - await testDetect(human, 'samples/ai-body.jpg', 'posenet'); + res = await testDetect(human, 'samples/ai-body.jpg', 'posenet'); + if (!res || res?.body?.length !== 1) log('error', 'failed: body posenet'); + else log('state', 'passed: body posenet'); config.body = { modelPath: 'movenet-lightning.json' }; - await testDetect(human, 'samples/ai-body.jpg', 'movenet'); + res = await testDetect(human, 'samples/ai-body.jpg', 'movenet'); + if (!res || res?.body?.length !== 1) log('error', 'failed: body movenet'); + else log('state', 'passed: body movenet'); + + // test handdetect and handtrack + log('info', 'test hand variants'); + config.hand = { enabled: true, maxDetected: 2, minConfidence: 0.1, detector: { modelPath: 'handdetect.json' } }; + res = await testDetect(human, 'samples/ai-body.jpg', 'handdetect'); + if (!res || res?.hand?.length !== 2) log('error', 'failed: hand handdetect'); + else log('state', 'passed: hand handdetect'); + config.hand = { enabled: true, maxDetected: 2, minConfidence: 0.1, detector: { modelPath: 'handtrack.json' } }; + res = await testDetect(human, 'samples/ai-body.jpg', 'handtrack'); + if (!res || res?.hand?.length !== 2) log('error', 'failed: hand handdetect'); + else log('state', 'passed: hand handdetect'); // test multiple instances const first = new Human(config);