diff --git a/CHANGELOG.md b/CHANGELOG.md index 737b367e..11d1064a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,10 @@ ## Changelog -### **HEAD -> main** 2021/09/24 mandic00@live.com +### **HEAD -> main** 2021/09/25 mandic00@live.com + + +### **origin/main** 2021/09/25 mandic00@live.com - new release diff --git a/samples/README.md b/samples/README.md index 4b96a893..9e53321c 100644 --- a/samples/README.md +++ b/samples/README.md @@ -3,7 +3,7 @@ Sample Images used by `Human` library demos and automated tests Not required for normal funcioning of library -Samples were generated using default configuration without any fine-tuning using command: +Samples were generated using command: ```shell node test/test-node-canvas.js samples/in/ samples/out/ diff --git a/samples/in/group-1.jpg b/samples/in/group-1.jpg index cb2cfead..2cf591fa 100644 Binary files a/samples/in/group-1.jpg and b/samples/in/group-1.jpg differ diff --git a/samples/in/group-3.jpg b/samples/in/group-3.jpg index b9c81560..5d0ea455 100644 Binary files a/samples/in/group-3.jpg and b/samples/in/group-3.jpg differ diff --git a/samples/in/group-4.jpg b/samples/in/group-4.jpg index c7033877..c59d75c4 100644 Binary files a/samples/in/group-4.jpg and b/samples/in/group-4.jpg differ diff --git a/samples/in/group-5.jpg b/samples/in/group-5.jpg index 35312594..2b7e7531 100644 Binary files a/samples/in/group-5.jpg and b/samples/in/group-5.jpg differ diff --git a/samples/people/christina.jpg b/samples/in/person-christina.jpg similarity index 100% rename from samples/people/christina.jpg rename to samples/in/person-christina.jpg diff --git a/samples/people/lauren.jpg b/samples/in/person-lauren.jpg similarity index 100% rename from samples/people/lauren.jpg rename to samples/in/person-lauren.jpg diff --git a/samples/people/lexi.jpg b/samples/in/person-lexi.jpg similarity index 100% rename from samples/people/lexi.jpg rename to samples/in/person-lexi.jpg diff --git a/samples/in/person-nicole.jpg b/samples/in/person-nicole.jpg new file mode 100644 index 00000000..ccf7bd28 Binary files /dev/null and b/samples/in/person-nicole.jpg differ diff --git a/samples/people/tasia.jpg b/samples/in/person-tasia.jpg similarity index 100% rename from samples/people/tasia.jpg rename to samples/in/person-tasia.jpg diff --git a/samples/people/vlado1.jpg b/samples/in/person-vlado1.jpg similarity index 100% rename from samples/people/vlado1.jpg rename to samples/in/person-vlado1.jpg diff --git a/samples/people/vlado5.jpg b/samples/in/person-vlado5.jpg similarity index 100% rename from samples/people/vlado5.jpg rename to samples/in/person-vlado5.jpg diff --git a/samples/out/ai-body.jpg b/samples/out/ai-body.jpg index 2ed46162..c6631ca3 100644 Binary files a/samples/out/ai-body.jpg and b/samples/out/ai-body.jpg differ diff --git a/samples/out/ai-face.jpg b/samples/out/ai-face.jpg index 989ce2a7..987f747f 100644 Binary files a/samples/out/ai-face.jpg and b/samples/out/ai-face.jpg differ diff --git a/samples/out/ai-upper.jpg b/samples/out/ai-upper.jpg index cd6d2a3d..84dc7ba7 100644 Binary files a/samples/out/ai-upper.jpg and b/samples/out/ai-upper.jpg differ diff --git a/samples/out/group-1.jpg b/samples/out/group-1.jpg index ceac1af3..c282680c 100644 Binary files a/samples/out/group-1.jpg and b/samples/out/group-1.jpg differ diff --git a/samples/out/group-2.jpg b/samples/out/group-2.jpg index 8b1e9344..1ac6ed45 100644 Binary files a/samples/out/group-2.jpg and b/samples/out/group-2.jpg differ diff --git a/samples/out/group-3.jpg b/samples/out/group-3.jpg index 6f2cd091..72d7096c 100644 Binary files a/samples/out/group-3.jpg and b/samples/out/group-3.jpg differ diff --git a/samples/out/group-4.jpg b/samples/out/group-4.jpg index 1bd2d4e4..6e4ddccb 100644 Binary files a/samples/out/group-4.jpg and b/samples/out/group-4.jpg differ diff --git a/samples/out/group-5.jpg b/samples/out/group-5.jpg index 66a44079..ab766365 100644 Binary files a/samples/out/group-5.jpg and b/samples/out/group-5.jpg differ diff --git a/samples/out/group-6.jpg b/samples/out/group-6.jpg index 60a82fa8..f04f407f 100644 Binary files a/samples/out/group-6.jpg and b/samples/out/group-6.jpg differ diff --git a/samples/out/group-7.jpg b/samples/out/group-7.jpg index 10ca4fea..44f8986f 100644 Binary files a/samples/out/group-7.jpg and b/samples/out/group-7.jpg differ diff --git a/samples/out/person-celeste.jpg b/samples/out/person-celeste.jpg index 5d796e99..5bbcd3bd 100644 Binary files a/samples/out/person-celeste.jpg and b/samples/out/person-celeste.jpg differ diff --git a/samples/out/person-christina.jpg b/samples/out/person-christina.jpg new file mode 100644 index 00000000..c80bb56e Binary files /dev/null and b/samples/out/person-christina.jpg differ diff --git a/samples/out/person-lauren.jpg b/samples/out/person-lauren.jpg new file mode 100644 index 00000000..ac06a6f0 Binary files /dev/null and b/samples/out/person-lauren.jpg differ diff --git a/samples/out/person-lexi.jpg b/samples/out/person-lexi.jpg new file mode 100644 index 00000000..af63cd2e Binary files /dev/null and b/samples/out/person-lexi.jpg differ diff --git a/samples/out/person-linda.jpg b/samples/out/person-linda.jpg index a3cd05a8..5c2ffa6d 100644 Binary files a/samples/out/person-linda.jpg and b/samples/out/person-linda.jpg differ diff --git a/samples/out/person-nicole.jpg b/samples/out/person-nicole.jpg new file mode 100644 index 00000000..e297eaa1 Binary files /dev/null and b/samples/out/person-nicole.jpg differ diff --git a/samples/out/person-tasia.jpg b/samples/out/person-tasia.jpg new file mode 100644 index 00000000..d1ca2615 Binary files /dev/null and b/samples/out/person-tasia.jpg differ diff --git a/samples/out/person-tetiana.jpg b/samples/out/person-tetiana.jpg index abdf4bbb..825a971f 100644 Binary files a/samples/out/person-tetiana.jpg and b/samples/out/person-tetiana.jpg differ diff --git a/samples/out/person-vlado.jpg b/samples/out/person-vlado.jpg index 3b948a38..b9f06b1e 100644 Binary files a/samples/out/person-vlado.jpg and b/samples/out/person-vlado.jpg differ diff --git a/samples/out/person-vlado1.jpg b/samples/out/person-vlado1.jpg new file mode 100644 index 00000000..feb78fc7 Binary files /dev/null and b/samples/out/person-vlado1.jpg differ diff --git a/samples/out/person-vlado5.jpg b/samples/out/person-vlado5.jpg new file mode 100644 index 00000000..b86a211d Binary files /dev/null and b/samples/out/person-vlado5.jpg differ diff --git a/samples/samples.html b/samples/samples.html index aa8c310c..deb165b5 100644 --- a/samples/samples.html +++ b/samples/samples.html @@ -9,34 +9,37 @@ - - - + + +
Human Examples Gallery
- +
diff --git a/src/config.ts b/src/config.ts index 25e4e313..6f4c828e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -71,8 +71,11 @@ export interface FaceConfig { * - minConfidence: threshold for discarding a prediction * - maxDetected: maximum number of people detected in the input, should be set to the minimum number for performance * + * `maxDetected` is valid for `posenet` and `movenet-multipose` as other models are single-pose only + * `maxDetected` can be set to -1 to auto-detect based on number of detected faces + * * Changing `modelPath` will change module responsible for hand detection and tracking - * Allowed values are 'posenet.json', 'blazepose.json', 'efficientpose.json', 'movenet-lightning.json', 'movenet-thunder.json', 'movenet-multipose.json' + * Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json` */ export interface BodyConfig { enabled: boolean, @@ -93,6 +96,8 @@ export interface BodyConfig { * - maxDetected: maximum number of hands detected in the input, should be set to the minimum number for performance * - rotation: use best-guess rotated hand image or just box with rotation as-is, false means higher performance, but incorrect finger mapping if hand is inverted * + * `maxDetected` can be set to -1 to auto-detect based on number of detected faces + * * Changing `detector.modelPath` will change module responsible for hand detection and tracking * Allowed values are `handdetect.json` and `handtrack.json` */ @@ -394,9 +399,10 @@ const config: Config = { enabled: true, modelPath: 'movenet-lightning.json', // body model, can be absolute path or relative to modelBasePath // can be 'posenet', 'blazepose', 'efficientpose', 'movenet-lightning', 'movenet-thunder' - maxDetected: 1, // maximum number of people detected in the input + maxDetected: -1, // maximum number of people detected in the input // should be set to the minimum number for performance - // only valid for posenet as other models detects single pose + // only valid for posenet and movenet-multipose as other models detects single pose + // set to -1 to autodetect based on number of detected faces minConfidence: 0.2, // threshold for discarding a prediction skipFrames: 1, // how many max frames to go without re-running the detector // only used when cacheSensitivity is not zero @@ -414,8 +420,9 @@ const config: Config = { // hasn't moved much in short time (10 * 1/25 = 0.25 sec) minConfidence: 0.8, // threshold for discarding a prediction iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed - maxDetected: 1, // maximum number of hands detected in the input + maxDetected: -1, // maximum number of hands detected in the input // should be set to the minimum number for performance + // set to -1 to autodetect based on number of detected faces landmarks: true, // detect hand landmarks or just hand boundary box detector: { modelPath: 'handdetect.json', // hand detector model, can be absolute path or relative to modelBasePath diff --git a/src/handtrack/handtrack.ts b/src/handtrack/handtrack.ts index 4fce40e0..d88fa840 100644 --- a/src/handtrack/handtrack.ts +++ b/src/handtrack/handtrack.ts @@ -15,6 +15,7 @@ import { env } from '../env'; import * as fingerPose from '../fingerpose/fingerpose'; import { fakeOps } from '../tfjs/backend'; +const boxScaleFact = 1.5; // hand finger model prefers slighly larger box const models: [GraphModel | null, GraphModel | null] = [null, null]; const modelOutputNodes = ['StatefulPartitionedCall/Postprocessor/Slice', 'StatefulPartitionedCall/Postprocessor/ExpandDims_1']; @@ -118,7 +119,15 @@ async function detectHands(input: Tensor, config: Config): Promise pt[0]), keypoints.map((pt) => pt[1])]; // all fingers coords const minmax = [Math.min(...finger[0]), Math.max(...finger[0]), Math.min(...finger[1]), Math.max(...finger[1])]; // find min and max coordinates for x and y of all fingers diff --git a/src/human.ts b/src/human.ts index 561404ed..067893ad 100644 --- a/src/human.ts +++ b/src/human.ts @@ -453,21 +453,24 @@ export class Human { if (elapsedTime > 0) this.performance.face = elapsedTime; } + if (this.config.async && (this.config.body.maxDetected === -1 || this.config.hand.maxDetected === -1)) faceRes = await faceRes; // need face result for auto-detect number of hands or bodies + // run body: can be posenet, blazepose, efficientpose, movenet this.analyze('Start Body:'); this.state = 'detect:body'; + const bodyConfig = this.config.body.maxDetected === -1 ? mergeDeep(this.config, { body: { maxDetected: 1 * (faceRes as FaceResult[]).length } }) : this.config; // autodetect number of bodies if (this.config.async) { - if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(img.tensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(img.tensor, bodyConfig) : []; if (this.performance.body) delete this.performance.body; } else { timeStamp = now(); - if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(img.tensor, this.config) : []; - else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(img.tensor, this.config) : []; + if (this.config.body.modelPath?.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(img.tensor, bodyConfig) : []; + else if (this.config.body.modelPath?.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(img.tensor, bodyConfig) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.body = elapsedTime; } @@ -476,14 +479,15 @@ export class Human { // run handpose this.analyze('Start Hand:'); this.state = 'detect:hand'; + const handConfig = this.config.hand.maxDetected === -1 ? mergeDeep(this.config, { hand: { maxDetected: 2 * (faceRes as FaceResult[]).length } }) : this.config; // autodetect number of hands if (this.config.async) { - if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? handpose.predict(img.tensor, this.config) : []; - else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? handtrack.predict(img.tensor, this.config) : []; + if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? handpose.predict(img.tensor, handConfig) : []; + else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? handtrack.predict(img.tensor, handConfig) : []; if (this.performance.hand) delete this.performance.hand; } else { timeStamp = now(); - if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? await handpose.predict(img.tensor, this.config) : []; - else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? await handtrack.predict(img.tensor, this.config) : []; + if (this.config.hand.detector?.modelPath?.includes('handdetect')) handRes = this.config.hand.enabled ? await handpose.predict(img.tensor, handConfig) : []; + else if (this.config.hand.detector?.modelPath?.includes('handtrack')) handRes = this.config.hand.enabled ? await handtrack.predict(img.tensor, handConfig) : []; elapsedTime = Math.trunc(now() - timeStamp); if (elapsedTime > 0) this.performance.hand = elapsedTime; } diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index 9e1ce554..122e6921 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -9,6 +9,7 @@ import * as tf from '../../dist/tfjs.esm.js'; import type { BodyResult } from '../result'; import type { GraphModel, Tensor } from '../tfjs/types'; import type { Config } from '../config'; +import { fakeOps } from '../tfjs/backend'; import { env } from '../env'; let model: GraphModel | null; @@ -27,6 +28,7 @@ const bodyParts = ['nose', 'leftEye', 'rightEye', 'leftEar', 'rightEar', 'leftSh export async function load(config: Config): Promise { if (env.initial) model = null; if (!model) { + fakeOps(['size'], config); model = await tf.loadGraphModel(join(config.modelBasePath, config.body.modelPath || '')) as unknown as GraphModel; if (!model || !model['modelUrl']) log('load model failed:', config.body.modelPath); else if (config.debug) log('load model:', model['modelUrl']); @@ -78,8 +80,8 @@ async function parseSinglePose(res, config, image) { async function parseMultiPose(res, config, image) { const persons: Array = []; - for (let p = 0; p < res[0].length; p++) { - const kpt = res[0][p]; + for (let id = 0; id < res[0].length; id++) { + const kpt = res[0][id]; score = Math.round(100 * kpt[51 + 4]) / 100; // eslint-disable-next-line no-continue if (score < config.body.minConfidence) continue; @@ -90,20 +92,14 @@ async function parseMultiPose(res, config, image) { keypoints.push({ part: bodyParts[i], score: partScore, - positionRaw: [ - kpt[3 * i + 1], - kpt[3 * i + 0], - ], - position: [ - Math.trunc(kpt[3 * i + 1] * (image.shape[2] || 0)), - Math.trunc(kpt[3 * i + 0] * (image.shape[1] || 0)), - ], + positionRaw: [kpt[3 * i + 1], kpt[3 * i + 0]], + position: [Math.trunc(kpt[3 * i + 1] * (image.shape[2] || 0)), Math.trunc(kpt[3 * i + 0] * (image.shape[1] || 0))], }); } } boxRaw = [kpt[51 + 1], kpt[51 + 0], kpt[51 + 3] - kpt[51 + 1], kpt[51 + 2] - kpt[51 + 0]]; persons.push({ - id: p, + id, score, boxRaw, box: [ @@ -112,7 +108,7 @@ async function parseMultiPose(res, config, image) { Math.trunc(boxRaw[2] * (image.shape[2] || 0)), Math.trunc(boxRaw[3] * (image.shape[1] || 0)), ], - keypoints, + keypoints: [...keypoints], }); } return persons; @@ -140,11 +136,11 @@ export async function predict(image: Tensor, config: Config): Promise { const decode = human.tf.node.decodeImage(buffer, 3);