From 8293bc26b9337b3b86c167921fae45d22d0dca61 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 7 Oct 2021 10:33:10 -0400 Subject: [PATCH] improve gaze and face angle visualizations in draw --- demo/index.js | 4 ++- src/tfjs/backend.ts | 10 +++---- src/util/draw.ts | 65 ++++++++++++++++++++++++++++++++++++--------- tfjs/tf-browser.ts | 2 +- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/demo/index.js b/demo/index.js index 19a52694..84ef723c 100644 --- a/demo/index.js +++ b/demo/index.js @@ -31,6 +31,8 @@ import jsonView from './helpers/jsonview.js'; let human; let userConfig = { + body: { enabled: false }, + hand: { enabled: false }, /* warmup: 'none', backend: 'humangl', @@ -408,7 +410,7 @@ async function setupCamera() { const track = stream.getVideoTracks()[0]; const settings = track.getSettings(); if (initialCameraAccess) log('selected video source:', track, settings); // log('selected camera:', track.label, 'id:', settings.deviceId); - ui.camera = { name: track.label.toLowerCase(), width: video.videoWidth, height: video.videoHeight, facing: settings.facingMode === 'user' ? 'front' : 'back' }; + ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' }; initialCameraAccess = false; if (!stream) return 'camera stream empty'; diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index 533461bc..813f71b5 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -51,7 +51,7 @@ export async function check(instance, force = false) { if (instance.config.debug) log('setting backend:', instance.config.backend); - // handle wasm + // customize wasm if (instance.config.backend === 'wasm') { if (instance.config.debug) log('wasm path:', instance.config.wasmPath); if (typeof tf?.setWasmPaths !== 'undefined') await tf.setWasmPaths(instance.config.wasmPath); @@ -71,13 +71,13 @@ export async function check(instance, force = false) { } } - // handle webgl & humangl + // customize humangl if (tf.getBackend() === 'humangl') { tf.ENV.set('CHECK_COMPUTATION_FOR_ERRORS', false); tf.ENV.set('WEBGL_CPU_FORWARD', true); tf.ENV.set('WEBGL_PACK_DEPTHWISECONV', false); tf.ENV.set('WEBGL_USE_SHAPES_UNIFORMS', true); - tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 128); + tf.ENV.set('CPU_HANDOFF_SIZE_THRESHOLD', 256); // if (!instance.config.object.enabled) tf.ENV.set('WEBGL_FORCE_F16_TEXTURES', true); // safe to use 16bit precision if (typeof instance.config['deallocate'] !== 'undefined' && instance.config['deallocate']) { // hidden param log('changing webgl: WEBGL_DELETE_TEXTURE_THRESHOLD:', true); @@ -89,9 +89,9 @@ export async function check(instance, force = false) { } } - // handle webgpu + // customize webgpu if (tf.getBackend() === 'webgpu') { - tf.ENV.set('WEBGPU_USE_GLSL', true); + tf.ENV.set('WEBGPU_CPU_HANDOFF_SIZE_THRESHOLD', 256); } // wait for ready diff --git a/src/util/draw.ts b/src/util/draw.ts index 60585961..f56bcf57 100644 --- a/src/util/draw.ts +++ b/src/util/draw.ts @@ -73,14 +73,14 @@ const getCanvasContext = (input) => { const rad2deg = (theta) => Math.round((theta * 180) / Math.PI); -function point(ctx, x, y, z = 0, localOptions) { +function point(ctx: CanvasRenderingContext2D, x, y, z = 0, localOptions) { ctx.fillStyle = localOptions.useDepth && z ? `rgba(${127.5 + (2 * z)}, ${127.5 - (2 * z)}, 255, 0.3)` : localOptions.color; ctx.beginPath(); ctx.arc(x, y, localOptions.pointSize, 0, 2 * Math.PI); ctx.fill(); } -function rect(ctx, x, y, width, height, localOptions) { +function rect(ctx: CanvasRenderingContext2D, x, y, width, height, localOptions) { ctx.beginPath(); if (localOptions.useCurves) { const cx = (x + x + width) / 2; @@ -102,7 +102,7 @@ function rect(ctx, x, y, width, height, localOptions) { ctx.stroke(); } -function lines(ctx, points: Point[] = [], localOptions) { +function lines(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) { if (points === undefined || points.length === 0) return; ctx.beginPath(); ctx.moveTo(points[0][0], points[0][1]); @@ -119,7 +119,7 @@ function lines(ctx, points: Point[] = [], localOptions) { } } -function curves(ctx, points: Point[] = [], localOptions) { +function curves(ctx: CanvasRenderingContext2D, points: Point[] = [], localOptions) { if (points === undefined || points.length === 0) return; if (!localOptions.useCurves || points.length <= 2) { lines(ctx, points, localOptions); @@ -139,6 +139,30 @@ function curves(ctx, points: Point[] = [], localOptions) { } } +function arrow(ctx: CanvasRenderingContext2D, from: Point, to: Point, radius = 5) { + let angle; + let x; + let y; + ctx.beginPath(); + ctx.moveTo(from[0], from[1]); + ctx.lineTo(to[0], to[1]); + angle = Math.atan2(to[1] - from[1], to[0] - from[0]); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.moveTo(x, y); + angle += (1.0 / 3.0) * (2 * Math.PI); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.lineTo(x, y); + angle += (1.0 / 3.0) * (2 * Math.PI); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.lineTo(x, y); + ctx.closePath(); + ctx.stroke(); + ctx.fill(); +} + export async function gesture(inCanvas: HTMLCanvasElement | OffscreenCanvas, result: Array, drawOptions?: Partial) { const localOptions = mergeDeep(options, drawOptions); if (!result || !inCanvas) return; @@ -242,25 +266,40 @@ export async function face(inCanvas: HTMLCanvasElement | OffscreenCanvas, result ctx.fill(); } } + if (localOptions.drawGaze && f.rotation?.angle) { + ctx.strokeStyle = 'pink'; + const valX = (f.box[0] + f.box[2] / 2) - (f.box[3] * rad2deg(f.rotation.angle.yaw) / 90); + const valY = (f.box[1] + f.box[3] / 2) + (f.box[2] * rad2deg(f.rotation.angle.pitch) / 90); + const pathV = new Path2D(` + M ${f.box[0] + f.box[2] / 2} ${f.box[1]} + C + ${valX} ${f.box[1]}, + ${valX} ${f.box[1] + f.box[3]}, + ${f.box[0] + f.box[2] / 2} ${f.box[1] + f.box[3]} + `); + const pathH = new Path2D(` + M ${f.box[0]} ${f.box[1] + f.box[3] / 2} + C + ${f.box[0]} ${valY}, + ${f.box[0] + f.box[2]} ${valY}, + ${f.box[0] + f.box[2]} ${f.box[1] + f.box[3] / 2} + `); + ctx.stroke(pathH); + ctx.stroke(pathV); + } if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.bearing && f.annotations['leftEyeIris'] && f.annotations['rightEyeIris'] && f.annotations['leftEyeIris'][0] && f.annotations['rightEyeIris'][0]) { ctx.strokeStyle = 'pink'; - ctx.beginPath(); - + ctx.fillStyle = 'pink'; const leftGaze = [ f.annotations['leftEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]), f.annotations['leftEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]), ]; - ctx.moveTo(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]); - ctx.lineTo(leftGaze[0], leftGaze[1]); - + arrow(ctx, [f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]], [leftGaze[0], leftGaze[1]], 4); const rightGaze = [ f.annotations['rightEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]), f.annotations['rightEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]), ]; - ctx.moveTo(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]); - ctx.lineTo(rightGaze[0], rightGaze[1]); - - ctx.stroke(); + arrow(ctx, [f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]], [rightGaze[0], rightGaze[1]], 4); } } } diff --git a/tfjs/tf-browser.ts b/tfjs/tf-browser.ts index bf51af3f..6d63dc48 100644 --- a/tfjs/tf-browser.ts +++ b/tfjs/tf-browser.ts @@ -32,7 +32,7 @@ export * from '@tensorflow/tfjs-backend-webgl/dist/index.js'; export * from '@tensorflow/tfjs-backend-wasm/dist/index.js'; // add webgpu to bundle, experimental -// export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js'; +export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js'; // export versions, overrides version object from @tensorflow/tfjs export { version } from '../dist/tfjs.version.js';