improve gaze and face angle visualizations in draw

demo/index.js

@@ -31,6 +31,8 @@ import jsonView from './helpers/jsonview.js';
 let human;
 
 let userConfig = {
+  body: { enabled: false },
+  hand: { enabled: false },
   /*
   warmup: 'none',
   backend: 'humangl',
@@ -408,7 +410,7 @@ async function setupCamera() {
   const track = stream.getVideoTracks()[0];
   const settings = track.getSettings();
   if (initialCameraAccess) log('selected video source:', track, settings); // log('selected camera:', track.label, 'id:', settings.deviceId);
-  ui.camera = { name: track.label.toLowerCase(), width: video.videoWidth, height: video.videoHeight, facing: settings.facingMode === 'user' ? 'front' : 'back' };
+  ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
   initialCameraAccess = false;
 
   if (!stream) return 'camera stream empty';

src/tfjs/backend.ts

@@ -51,7 +51,7 @@ export async function check(instance, force = false) {
 
       if (instance.config.debug) log('setting backend:', instance.config.backend);
 
-      // handle wasm
+      // customize wasm
       if (instance.config.backend === 'wasm') {
         if (instance.config.debug) log('wasm path:', instance.config.wasmPath);
         if (typeof tf?.setWasmPaths !== 'undefined') await tf.setWasmPaths(instance.config.wasmPath);
@@ -71,13 +71,13 @@ export async function check(instance, force = false) {
       }
     }
 
-    // handle webgl & humangl
+    // customize humangl
     if (tf.getBackend() === 'humangl') {
       tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false);
       tf.ENV.set('WEBGL_CPU_FORWARD', true);
       tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', false);
       tf.ENV.set('WEBGL_USE_SHAPES_UNIFORMS', true);
-      tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 128);
+      tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 256);
       // if (!instance.config.object.enabled) tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // safe to use 16bit precision
       if (typeof instance.config['deallocate'] !== 'undefined' && instance.config['deallocate']) { // hidden param
         log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true);
@@ -89,9 +89,9 @@ export async function check(instance, force = false) {
       }
     }
 
-    // handle webgpu
+    // customize webgpu
     if (tf.getBackend() === 'webgpu') {
-      tf.ENV.set('WEBGPU_USE_GLSL', true);
+      tf.ENV.set('WEBGPU_CPU_HANDOFF_SIZE_THRESHOLD', 256);
     }
 
     // wait for ready

src/util/draw.ts

@@ -73,14 +73,14 @@ const getCanvasContext = (input) => {
 
 const rad2deg = (theta) => Math.round((theta * 180) / Math.PI);
 
-function point(ctx, x, y, z = 0, localOptions) {
+function point(ctx: CanvasRenderingContext2D, x, y, z = 0, localOptions) {
   ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color;
   ctx.beginPath();
   ctx.arc(x, y, localOptions.pointSize, 0, 2 * Math.PI);
   ctx.fill();
 }
 
-function rect(ctx, x, y, width, height, localOptions) {
+function rect(ctx: CanvasRenderingContext2D, x, y, width, height, localOptions) {
   ctx.beginPath();
   if (localOptions.useCurves) {
     const cx = (x + x + width) / 2;
@@ -102,7 +102,7 @@ function rect(ctx, x, y, width, height, localOptions) {
   ctx.stroke();
 }
 
-function lines(ctx, points: Point[] = [], localOptions) {
+function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
   if (points === undefined || points.length === 0) return;
   ctx.beginPath();
   ctx.moveTo(points[0][0], points[0][1]);
@@ -119,7 +119,7 @@ function lines(ctx, points: Point[] = [], localOptions) {
   }
 }
 
-function curves(ctx, points: Point[] = [], localOptions) {
+function curves(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) {
   if (points === undefined || points.length === 0) return;
   if (!localOptions.useCurves || points.length <= 2) {
     lines(ctx, points, localOptions);
@@ -139,6 +139,30 @@ function curves(ctx, points: Point[] = [], localOptions) {
   }
 }
 
+function arrow(ctx: CanvasRenderingContext2D, from: Point, to: Point, radius = 5) {
+  let angle;
+  let x;
+  let y;
+  ctx.beginPath();
+  ctx.moveTo(from[0], from[1]);
+  ctx.lineTo(to[0], to[1]);
+  angle = Math.atan2(to[1] - from[1], to[0] - from[0]);
+  x = radius * Math.cos(angle) + to[0];
+  y = radius * Math.sin(angle) + to[1];
+  ctx.moveTo(x, y);
+  angle += (1.0 / 3.0) * (2 * Math.PI);
+  x = radius * Math.cos(angle) + to[0];
+  y = radius * Math.sin(angle) + to[1];
+  ctx.lineTo(x, y);
+  angle += (1.0 / 3.0) * (2 * Math.PI);
+  x = radius * Math.cos(angle) + to[0];
+  y = radius * Math.sin(angle) + to[1];
+  ctx.lineTo(x, y);
+  ctx.closePath();
+  ctx.stroke();
+  ctx.fill();
+}
+
 export async function gesture(inCanvas: HTMLCanvasElement | OffscreenCanvas, result: Array<GestureResult>, drawOptions?: Partial<DrawOptions>) {
   const localOptions = mergeDeep(options, drawOptions);
   if (!result || !inCanvas) return;
@@ -242,25 +266,40 @@ export async function face(inCanvas: HTMLCanvasElement | OffscreenCanvas, result
             ctx.fill();
           }
         }
+        if (localOptions.drawGaze && f.rotation?.angle) {
+          ctx.strokeStyle = 'pink';
+          const valX = (f.box[0] + f.box[2] / 2) - (f.box[3] * rad2deg(f.rotation.angle.yaw) / 90);
+          const valY = (f.box[1] + f.box[3] / 2) + (f.box[2] * rad2deg(f.rotation.angle.pitch) / 90);
+          const pathV = new Path2D(`
+            M ${f.box[0] + f.box[2] / 2} ${f.box[1]}
+            C
+              ${valX} ${f.box[1]},
+              ${valX} ${f.box[1] + f.box[3]},
+              ${f.box[0] + f.box[2] / 2} ${f.box[1] + f.box[3]}
+          `);
+          const pathH = new Path2D(`
+            M ${f.box[0]} ${f.box[1] + f.box[3] / 2}
+            C 
+              ${f.box[0]} ${valY},
+              ${f.box[0] + f.box[2]} ${valY},
+              ${f.box[0] + f.box[2]} ${f.box[1] + f.box[3] / 2}
+          `);
+          ctx.stroke(pathH);
+          ctx.stroke(pathV);
+        }
         if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.bearing && f.annotations['leftEyeIris'] && f.annotations['rightEyeIris'] && f.annotations['leftEyeIris'][0] && f.annotations['rightEyeIris'][0]) {
           ctx.strokeStyle = 'pink';
-          ctx.beginPath();
+          ctx.fillStyle = 'pink';
-
           const leftGaze = [
             f.annotations['leftEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
             f.annotations['leftEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
           ];
-          ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]);
+          arrow(ctx, [f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]], [leftGaze[0], leftGaze[1]], 4);
-          ctx.lineTo(leftGaze[0], leftGaze[1]);
-
           const rightGaze = [
             f.annotations['rightEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]),
             f.annotations['rightEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]),
           ];
-          ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]);
+          arrow(ctx, [f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]], [rightGaze[0], rightGaze[1]], 4);
-          ctx.lineTo(rightGaze[0], rightGaze[1]);
-
-          ctx.stroke();
         }
       }
     }

tfjs/tf-browser.ts

@@ -32,7 +32,7 @@ export * from '@tensorflow/tfjs-backend-webgl/dist/index.js';
 export * from '@tensorflow/tfjs-backend-wasm/dist/index.js';
 
 // add webgpu to bundle, experimental
-// export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js';
+export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js';
 
 // export versions, overrides version object from @tensorflow/tfjs
 export { version } from '../dist/tfjs.version.js';