implement event emitters

README.md

@@ -195,11 +195,11 @@ draw output on screen using internal draw helper functions
 // create instance of human with simple configuration using default values
 const config = { backend: 'webgl' };
 const human = new Human(config);
+// select input HTMLVideoElement and output HTMLCanvasElement from page
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
 
 function detectVideo() {
-  // select input HTMLVideoElement and output HTMLCanvasElement from page
-  const inputVideo = document.getElementById('video-id');
-  const outputCanvas = document.getElementById('canvas-id');
   // perform processing using default configuration
   human.detect(inputVideo).then((result) => {
     // result object will contain detected details
@@ -225,10 +225,10 @@ or using `async/await`:
 // create instance of human with simple configuration using default values
 const config = { backend: 'webgl' };
 const human = new Human(config); // create instance of Human
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
 
 async function detectVideo() {
-  const inputVideo = document.getElementById('video-id');
-  const outputCanvas = document.getElementById('canvas-id');
   const result = await human.detect(inputVideo); // run detection
   human.draw.all(outputCanvas, result); // draw all results
   requestAnimationFrame(detectVideo); // run loop
@@ -237,6 +237,27 @@ async function detectVideo() {
 detectVideo(); // start loop
 ```
 
+or using `Events`:
+
+```js
+// create instance of human with simple configuration using default values
+const config = { backend: 'webgl' };
+const human = new Human(config); // create instance of Human
+const inputVideo = document.getElementById('video-id');
+const outputCanvas = document.getElementById('canvas-id');
+
+human.events.addEventListener('detect', () => { // event gets triggered when detect is complete
+  human.draw.all(outputCanvas, human.result); // draw all results
+});
+
+function detectVideo() {
+  human.detect(inputVideo) // run detection
+  .then(() => requestAnimationFrame(detectVideo)); // upon detect complete start processing of the next frame
+}
+
+detectVideo(); // start loop
+```
+
 or using interpolated results for smooth video processing by separating detection and drawing loops:
 
 ```js

demo/nodejs/node-canvas.js

@@ -7,8 +7,6 @@ const fs = require('fs');
 const process = require('process');
 const canvas = require('canvas');
 
-let fetch; // fetch is dynamically imported later
-
 // for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
 // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
 const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
@@ -51,25 +49,26 @@ async function init() {
 }
 
 async function detect(input, output) {
-  // read input image file and create tensor to be used for processing
+  // read input image from file or url into buffer
   let buffer;
   log.info('Loading image:', input);
   if (input.startsWith('http:') || input.startsWith('https:')) {
+    const fetch = (await import('node-fetch')).default;
     const res = await fetch(input);
     if (res && res.ok) buffer = await res.buffer();
     else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
   } else {
     buffer = fs.readFileSync(input);
   }
+  if (!buffer) return {};
 
   // decode image using tfjs-node so we don't need external depenencies
-  // can also be done using canvas.js or some other 3rd party image library
+  /*
-  if (!buffer) return {};
   const tensor = human.tf.tidy(() => {
     const decode = human.tf.node.decodeImage(buffer, 3);
     let expand;
     if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
-      const channels = human.tf.split(decode, 4, 2); // tf.split(tensor, 4, 2); // split rgba to channels
+      const channels = human.tf.split(decode, 4, 2); // split rgba to channels
       const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
       expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
     } else {
@@ -78,6 +77,22 @@ async function detect(input, output) {
     const cast = human.tf.cast(expand, 'float32');
     return cast;
   });
+  */
+
+  // decode image using canvas library
+  const inputImage = await canvas.loadImage(input);
+  const inputCanvas = new canvas.Canvas(inputImage.width, inputImage.height, 'image');
+  const inputCtx = inputCanvas.getContext('2d');
+  inputCtx.drawImage(inputImage, 0, 0);
+  const inputData = inputCtx.getImageData(0, 0, inputImage.width, inputImage.height);
+  const tensor = human.tf.tidy(() => {
+    const data = tf.tensor(Array.from(inputData.data), [inputImage.width, inputImage.height, 4]);
+    const channels = human.tf.split(data, 4, 2); // split rgba to channels
+    const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
+    const expand = human.tf.reshape(rgb, [1, data.shape[0], data.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
+    const cast = human.tf.cast(expand, 'float32');
+    return cast;
+  });
 
   // image shape contains image dimensions and depth
   log.state('Processing:', tensor['shape']);
@@ -130,7 +145,6 @@ async function detect(input, output) {
 async function main() {
   log.header();
   log.info('Current folder:', process.env.PWD);
-  fetch = (await import('node-fetch')).default;
   await init();
   const input = process.argv[2];
   const output = process.argv[3];

demo/nodejs/node-event.js

@@ -0,0 +1,110 @@
+/**
+ * Human demo for NodeJS
+ */
+
+const log = require('@vladmandic/pilogger');
+const fs = require('fs');
+const process = require('process');
+
+let fetch; // fetch is dynamically imported later
+
+// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
+// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
+
+// load specific version of Human library that matches TensorFlow mode
+const Human = require('../../dist/human.node.js').default; // or const Human = require('../dist/human.node-gpu.js').default;
+
+let human = null;
+
+const myConfig = {
+  backend: 'tensorflow',
+  modelBasePath: 'file://models/',
+  debug: false,
+  async: true,
+  filter: { enabled: false },
+  face: {
+    enabled: true,
+    detector: { enabled: true },
+    mesh: { enabled: true },
+    iris: { enabled: true },
+    description: { enabled: true },
+    emotion: { enabled: true },
+  },
+  hand: { enabled: true },
+  body: { enabled: true },
+  object: { enabled: true },
+};
+
+async function detect(input) {
+  // read input image from file or url into buffer
+  let buffer;
+  log.info('Loading image:', input);
+  if (input.startsWith('http:') || input.startsWith('https:')) {
+    fetch = (await import('node-fetch')).default;
+    const res = await fetch(input);
+    if (res && res.ok) buffer = await res.buffer();
+    else log.error('Invalid image URL:', input, res.status, res.statusText, res.headers.get('content-type'));
+  } else {
+    buffer = fs.readFileSync(input);
+  }
+
+  // decode image using tfjs-node so we don't need external depenencies
+  if (!buffer) return;
+  const tensor = human.tf.tidy(() => {
+    const decode = human.tf.node.decodeImage(buffer, 3);
+    let expand;
+    if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
+      const channels = human.tf.split(decode, 4, 2); // split rgba to channels
+      const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
+      expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
+    } else {
+      expand = human.tf.expandDims(decode, 0);
+    }
+    const cast = human.tf.cast(expand, 'float32');
+    return cast;
+  });
+
+  // run detection
+  await human.detect(tensor, myConfig);
+  human.tf.dispose(tensor); // dispose image tensor as we no longer need it
+}
+
+async function main() {
+  log.header();
+
+  human = new Human(myConfig);
+
+  human.events.addEventListener('warmup', () => {
+    log.info('Event Warmup');
+  });
+
+  human.events.addEventListener('load', () => {
+    const loaded = Object.keys(human.models).filter((a) => human.models[a]);
+    log.info('Event Loaded:', loaded, human.tf.engine().memory());
+  });
+
+  human.events.addEventListener('image', () => {
+    log.info('Event Image:', human.process.tensor.shape);
+  });
+
+  human.events.addEventListener('detect', () => {
+    log.data('Event Detected:');
+    const persons = human.result.persons;
+    for (let i = 0; i < persons.length; i++) {
+      const face = persons[i].face;
+      const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
+      const body = persons[i].body;
+      const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
+      log.data(`  #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
+    }
+  });
+
+  await human.tf.ready(); // wait until tf is ready
+
+  const input = process.argv[2]; // process input
+  if (input) await detect(input);
+  else log.error('Missing <input>');
+}
+
+main();

src/human.ts

@@ -76,8 +76,10 @@ export class Human {
    * - Progresses through: 'config', 'check', 'backend', 'load', 'run:<model>', 'idle'
    */
   state: string;
-  /** @internal: Instance of current image being processed */
+  /** process input and return tensor and canvas */
-  image: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
+  image: typeof image.process;
+  /** currenty processed image tensor and canvas */
+  process: { tensor: Tensor | null, canvas: OffscreenCanvas | HTMLCanvasElement | null };
   /** @internal: Instance of TensorFlow/JS used by Human
    * - Can be embedded or externally provided
    */
@@ -87,7 +89,7 @@ export class Human {
    * - face: draw detected faces
    * - body: draw detected people and body parts
    * - hand: draw detected hands and hand parts
-   * - canvas: draw processed canvas which is a processed copy of the input
+   * - canvas: draw this.processed canvas which is a this.processed copy of the input
    * - all: meta-function that performs: canvas, face, body, hand
    */
   draw: {
@@ -126,6 +128,17 @@ export class Human {
     faceres: GraphModel | null,
     segmentation: GraphModel | null,
   };
+  /** Container for events dispatched by Human
+   *
+   * Possible events:
+   * - `create`: triggered when Human object is instantiated
+   * - `load`: triggered when models are loaded (explicitly or on-demand)
+   * - `image`: triggered when input image is this.processed
+   * - `result`: triggered when detection is complete
+   * - `warmup`: triggered when warmup is complete
+   */
+
+  events: EventTarget;
   /** Reference face triangualtion array of 468 points, used for triangle references between points */
   faceTriangulation: typeof facemesh.triangulation;
   /** Refernce UV map of 468 values, used for 3D mapping of the face mesh */
@@ -161,6 +174,7 @@ export class Human {
     this.#firstRun = true;
     this.#lastCacheDiff = 0;
     this.performance = { backend: 0, load: 0, image: 0, frames: 0, cached: 0, changed: 0, total: 0, draw: 0 };
+    this.events = new EventTarget();
     // object that contains all initialized models
     this.models = {
       face: null,
@@ -179,15 +193,17 @@ export class Human {
       segmentation: null,
     };
     this.result = { face: [], body: [], hand: [], gesture: [], object: [], performance: {}, timestamp: 0, persons: [] };
-    // export access to image processing
+    // export access to image this.processing
     // @ts-ignore eslint-typescript cannot correctly infer type in anonymous function
     this.image = (input: Input) => image.process(input, this.config);
+    this.process = { tensor: null, canvas: null };
     // export raw access to underlying models
     this.faceTriangulation = facemesh.triangulation;
     this.faceUVMap = facemesh.uvmap;
     // include platform info
     this.sysinfo = sysinfo.info();
     this.#lastInputSum = 1;
+    this.#emit('create');
   }
 
   // helper function: measure tensor leak
@@ -228,9 +244,9 @@ export class Human {
   }
 
   /**
-   * Segmentation method takes any input and returns processed canvas with body segmentation
+   * Segmentation method takes any input and returns this.processed canvas with body segmentation
    * Optional parameter background is used to fill the background with specific input
-   * Segmentation is not triggered as part of detect process
+   * Segmentation is not triggered as part of detect this.process
    *
    * @param input: {@link Input}
    * @param background?: {@link Input}
@@ -240,7 +256,7 @@ export class Human {
     return segmentation.process(input, background, this.config);
   }
 
-  /** Enhance method performs additional enhacements to face image previously detected for futher processing
+  /** Enhance method performs additional enhacements to face image previously detected for futher this.processing
    * @param input: Tensor as provided in human.result.face[n].tensor
    * @returns Tensor
    */
@@ -267,6 +283,7 @@ export class Human {
   async load(userConfig?: Config | Record<string, unknown>) {
     this.state = 'load';
     const timeStamp = now();
+    const count = Object.values(this.models).filter((model) => model).length;
     if (userConfig) this.config = mergeDeep(this.config, userConfig) as Config;
 
     if (this.#firstRun) { // print version info on first run and check for correct backend setup
@@ -289,10 +306,16 @@ export class Human {
       this.#firstRun = false;
     }
 
+    const loaded = Object.values(this.models).filter((model) => model).length;
+    if (loaded !== count) this.#emit('load');
     const current = Math.trunc(now() - timeStamp);
     if (current > (this.performance.load as number || 0)) this.performance.load = current;
   }
 
+  // emit event
+  /** @hidden */
+  #emit = (event: string) => this.events?.dispatchEvent(new Event(event));
+
   // check if backend needs initialization if it changed
   /** @hidden */
   #checkBackend = async (force = false) => {
@@ -433,9 +456,9 @@ export class Human {
 
   /** Main detection method
    * - Analyze configuration: {@link Config}
-   * - Pre-process input: {@link Input}
+   * - Pre-this.process input: {@link Input}
    * - Run inference for all configured models
-   * - Process and return result: {@link Result}
+   * - this.process and return result: {@link Result}
    *
    * @param input: Input
    * @param userConfig?: {@link Config}
@@ -468,34 +491,35 @@ export class Human {
       await this.load();
 
       timeStamp = now();
-      let process = image.process(input, this.config);
+      this.process = image.process(input, this.config);
       this.performance.image = Math.trunc(now() - timeStamp);
       this.analyze('Get Image:');
 
-      // run segmentation preprocessing
+      // run segmentation prethis.processing
-      if (this.config.segmentation.enabled && process && process.tensor) {
+      if (this.config.segmentation.enabled && this.process && this.process.tensor) {
         this.analyze('Start Segmentation:');
         this.state = 'run:segmentation';
         timeStamp = now();
-        await segmentation.predict(process);
+        await segmentation.predict(this.process);
         elapsedTime = Math.trunc(now() - timeStamp);
         if (elapsedTime > 0) this.performance.segmentation = elapsedTime;
-        if (process.canvas) {
+        if (this.process.canvas) {
           // replace input
-          tf.dispose(process.tensor);
+          tf.dispose(this.process.tensor);
-          process = image.process(process.canvas, this.config);
+          this.process = image.process(this.process.canvas, this.config);
         }
         this.analyze('End Segmentation:');
       }
 
-      if (!process || !process.tensor) {
+      if (!this.process || !this.process.tensor) {
         log('could not convert input to tensor');
         resolve({ error: 'could not convert input to tensor' });
         return;
       }
+      this.#emit('image');
 
       timeStamp = now();
-      this.config.skipFrame = await this.#skipFrame(process.tensor);
+      this.config.skipFrame = await this.#skipFrame(this.process.tensor);
       if (!this.performance.frames) this.performance.frames = 0;
       if (!this.performance.cached) this.performance.cached = 0;
       (this.performance.frames as number)++;
@@ -512,12 +536,12 @@ export class Human {
 
       // run face detection followed by all models that rely on face bounding box: face mesh, age, gender, emotion
       if (this.config.async) {
-        faceRes = this.config.face.enabled ? face.detectFace(this, process.tensor) : [];
+        faceRes = this.config.face.enabled ? face.detectFace(this, this.process.tensor) : [];
         if (this.performance.face) delete this.performance.face;
       } else {
         this.state = 'run:face';
         timeStamp = now();
-        faceRes = this.config.face.enabled ? await face.detectFace(this, process.tensor) : [];
+        faceRes = this.config.face.enabled ? await face.detectFace(this, this.process.tensor) : [];
         elapsedTime = Math.trunc(now() - timeStamp);
         if (elapsedTime > 0) this.performance.face = elapsedTime;
       }
@@ -525,18 +549,18 @@ export class Human {
       // run body: can be posenet, blazepose, efficientpose, movenet
       this.analyze('Start Body:');
       if (this.config.async) {
-        if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(process.tensor, this.config) : [];
+        if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? posenet.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? blazepose.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? efficientpose.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? movenet.predict(this.process.tensor, this.config) : [];
         if (this.performance.body) delete this.performance.body;
       } else {
         this.state = 'run:body';
         timeStamp = now();
-        if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(process.tensor, this.config) : [];
+        if (this.config.body.modelPath.includes('posenet')) bodyRes = this.config.body.enabled ? await posenet.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('blazepose')) bodyRes = this.config.body.enabled ? await blazepose.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('efficientpose')) bodyRes = this.config.body.enabled ? await efficientpose.predict(this.process.tensor, this.config) : [];
-        else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(process.tensor, this.config) : [];
+        else if (this.config.body.modelPath.includes('movenet')) bodyRes = this.config.body.enabled ? await movenet.predict(this.process.tensor, this.config) : [];
         elapsedTime = Math.trunc(now() - timeStamp);
         if (elapsedTime > 0) this.performance.body = elapsedTime;
       }
@@ -545,12 +569,12 @@ export class Human {
       // run handpose
       this.analyze('Start Hand:');
       if (this.config.async) {
-        handRes = this.config.hand.enabled ? handpose.predict(process.tensor, this.config) : [];
+        handRes = this.config.hand.enabled ? handpose.predict(this.process.tensor, this.config) : [];
         if (this.performance.hand) delete this.performance.hand;
       } else {
         this.state = 'run:hand';
         timeStamp = now();
-        handRes = this.config.hand.enabled ? await handpose.predict(process.tensor, this.config) : [];
+        handRes = this.config.hand.enabled ? await handpose.predict(this.process.tensor, this.config) : [];
         elapsedTime = Math.trunc(now() - timeStamp);
         if (elapsedTime > 0) this.performance.hand = elapsedTime;
       }
@@ -559,14 +583,14 @@ export class Human {
       // run nanodet
       this.analyze('Start Object:');
       if (this.config.async) {
-        if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(process.tensor, this.config) : [];
+        if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? nanodet.predict(this.process.tensor, this.config) : [];
-        else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(process.tensor, this.config) : [];
+        else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? centernet.predict(this.process.tensor, this.config) : [];
         if (this.performance.object) delete this.performance.object;
       } else {
         this.state = 'run:object';
         timeStamp = now();
-        if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(process.tensor, this.config) : [];
+        if (this.config.object.modelPath.includes('nanodet')) objectRes = this.config.object.enabled ? await nanodet.predict(this.process.tensor, this.config) : [];
-        else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(process.tensor, this.config) : [];
+        else if (this.config.object.modelPath.includes('centernet')) objectRes = this.config.object.enabled ? await centernet.predict(this.process.tensor, this.config) : [];
         elapsedTime = Math.trunc(now() - timeStamp);
         if (elapsedTime > 0) this.performance.object = elapsedTime;
       }
@@ -586,6 +610,7 @@ export class Human {
 
       this.performance.total = Math.trunc(now() - timeStart);
       this.state = 'idle';
+      const shape = this.process?.tensor?.shape || [];
       this.result = {
         face: faceRes as Face[],
         body: bodyRes as Body[],
@@ -593,15 +618,16 @@ export class Human {
         gesture: gestureRes,
         object: objectRes as Item[],
         performance: this.performance,
-        canvas: process.canvas,
+        canvas: this.process.canvas,
         timestamp: Date.now(),
-        get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, process?.tensor?.shape); },
+        get persons() { return persons.join(faceRes as Face[], bodyRes as Body[], handRes as Hand[], gestureRes, shape); },
       };
 
       // finally dispose input tensor
-      tf.dispose(process.tensor);
+      tf.dispose(this.process.tensor);
 
       // log('Result:', result);
+      this.#emit('detect');
       resolve(this.result);
     });
   }
@@ -700,6 +726,7 @@ export class Human {
     else res = await this.#warmupNode();
     const t1 = now();
     if (this.config.debug) log('Warmup', this.config.warmup, Math.round(t1 - t0), 'ms', res);
+    this.#emit('warmup');
     return res;
   }
 }

src/result.ts

@@ -186,7 +186,7 @@ export interface Result {
   /** global performance object with timing values for each operation */
   performance: Record<string, unknown>,
   /** optional processed canvas that can be used to draw input on screen */
-  canvas?: OffscreenCanvas | HTMLCanvasElement,
+  canvas?: OffscreenCanvas | HTMLCanvasElement | null,
   /** timestamp of detection representing the milliseconds elapsed since the UNIX epoch */
   readonly timestamp: number,
   /** getter property that returns unified persons object  */