From 08f8c6af15c56579248128e420a9c1d6a399f338 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 13 Oct 2020 09:59:21 -0400 Subject: [PATCH] updated demo --- .eslintrc.json | 1 + demo/demo-esm.html | 2 +- demo/demo-esm.js | 140 ++++++++++++++++++++++++++---------------- demo/demo-iife.js | 1 - src/handpose/hand.js | 6 +- src/handpose/index.js | 12 ---- 6 files changed, 90 insertions(+), 72 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index e8937215..83e32294 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -39,6 +39,7 @@ "no-restricted-globals": "off", "no-restricted-syntax": "off", "no-underscore-dangle": "off", + "no-return-assign": "off", "newline-per-chained-call": "off", "node/no-unsupported-features/es-syntax": "off", "node/shebang": "off", diff --git a/demo/demo-esm.html b/demo/demo-esm.html index c8aa84a9..f49f42e2 100644 --- a/demo/demo-esm.html +++ b/demo/demo-esm.html @@ -1,5 +1,5 @@ - + diff --git a/demo/demo-esm.js b/demo/demo-esm.js index c3dfc9af..3e9b4a9a 100644 --- a/demo/demo-esm.js +++ b/demo/demo-esm.js @@ -1,8 +1,13 @@ /* global QuickSettings */ -/* eslint-disable no-return-assign */ import human from '../dist/human.esm.js'; +const ui = { + baseColor: 'rgba(255, 200, 255, 0.3)', + baseFont: 'small-caps 1.2rem "Segoe UI"', + baseLineWidth: 16, +}; + const config = { face: { enabled: true, @@ -17,17 +22,20 @@ const config = { }; let settings; -async function drawFace(result) { - const canvas = document.getElementById('canvas'); +async function drawFace(result, canvas) { const ctx = canvas.getContext('2d'); - ctx.fillStyle = 'lightcoral'; - ctx.strokeStyle = 'lightcoral'; - ctx.font = 'small-caps 1rem "Segoe UI"'; + ctx.fillStyle = ui.baseColor; + ctx.strokeStyle = ui.baseColor; + ctx.font = ui.baseFont; for (const face of result) { + ctx.lineWidth = ui.baseLineWidth; ctx.beginPath(); - ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]); - ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 16, face.box[2]); + if (settings.getValue('Draw Boxes')) { + ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]); + } + ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]); ctx.stroke(); + ctx.lineWidth = 1; if (face.mesh) { if (settings.getValue('Draw Points')) { for (const point of face.mesh) { @@ -50,10 +58,10 @@ async function drawFace(result) { path.lineTo(point[0], point[1]); } path.closePath(); - ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.5)`; - ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.5)`; + ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`; ctx.stroke(path); if (settings.getValue('Fill Polygons')) { + ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`; ctx.fill(path); } } @@ -62,12 +70,12 @@ async function drawFace(result) { } } -async function drawBody(result) { - const canvas = document.getElementById('canvas'); +async function drawBody(result, canvas) { const ctx = canvas.getContext('2d'); - ctx.fillStyle = 'lightcoral'; - ctx.strokeStyle = 'lightcoral'; - ctx.font = 'small-caps 1rem "Segoe UI"'; + ctx.fillStyle = ui.baseColor; + ctx.strokeStyle = ui.baseColor; + ctx.font = ui.baseFont; + ctx.lineWidth = ui.baseLineWidth; for (const pose of result) { if (settings.getValue('Draw Points')) { for (const point of pose.keypoints) { @@ -123,12 +131,19 @@ async function drawBody(result) { } } -async function drawHand(result) { - const canvas = document.getElementById('canvas'); +async function drawHand(result, canvas) { const ctx = canvas.getContext('2d'); - ctx.font = 'small-caps 1rem "Segoe UI"'; + ctx.font = ui.baseFont; + ctx.lineWidth = ui.baseLineWidth; window.result = result; for (const hand of result) { + if (settings.getValue('Draw Boxes')) { + ctx.lineWidth = ui.baseLineWidth; + ctx.beginPath(); + ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]); + ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]); + ctx.stroke(); + } if (settings.getValue('Draw Points')) { for (const point of hand.landmarks) { ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`; @@ -139,13 +154,14 @@ async function drawHand(result) { } if (settings.getValue('Draw Polygons')) { const addPart = (part) => { - ctx.beginPath(); - for (const i in part) { + for (let i = 1; i < part.length; i++) { + ctx.lineWidth = ui.baseLineWidth; + ctx.beginPath(); ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`; - if (i === 0) ctx.moveTo(part[i][0], part[i][1]); - else ctx.lineTo(part[i][0], part[i][1]); + ctx.moveTo(part[i - 1][0], part[i - 1][1]); + ctx.lineTo(part[i][0], part[i][1]); + ctx.stroke(); } - ctx.stroke(); }; addPart(hand.annotations.indexFinger); addPart(hand.annotations.middleFinger); @@ -157,25 +173,30 @@ async function drawHand(result) { } } -async function runHumanDetect() { - const video = document.getElementById('video'); - const canvas = document.getElementById('canvas'); +async function runHumanDetect(input, canvas) { const log = document.getElementById('log'); - const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false; - if (live) { + const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false; + // perform detect if live video or not video at all + if (live || !(input instanceof HTMLVideoElement)) { // perform detection const t0 = performance.now(); - const result = await human.detect(video, config); + let result; + try { + result = await human.detect(input, config); + } catch (err) { + log.innerText = err.message; + } + if (!result) return; const t1 = performance.now(); // update fps settings.setValue('FPS', Math.round(1000 / (t1 - t0))); // draw image from video const ctx = canvas.getContext('2d'); - ctx.drawImage(video, 0, 0, video.width, video.height, 0, 0, canvas.width, canvas.height); + ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height); // draw all results - drawFace(result.face); - drawBody(result.body); - drawHand(result.hand); + drawFace(result.face, canvas); + drawBody(result.body, canvas); + drawHand(result.hand, canvas); // update log const engine = await human.tf.engine(); log.innerText = ` @@ -184,12 +205,13 @@ async function runHumanDetect() { Result Object Size: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes `; // rinse & repeate - // setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes - requestAnimationFrame(runHumanDetect); // immediate loop + // if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes + if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop } } function setupGUI() { + // add all variables to ui control panel settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); settings.addRange('FPS', 0, 100, 0, 1); settings.addBoolean('Pause', false, (val) => { @@ -198,9 +220,13 @@ function setupGUI() { runHumanDetect(); }); settings.addHTML('line1', '
'); settings.hideTitle('line1'); + settings.addBoolean('Draw Boxes', false); settings.addBoolean('Draw Points', true); settings.addBoolean('Draw Polygons', true); settings.addBoolean('Fill Polygons', true); + settings.bindText('baseColor', ui.baseColor, config); + settings.bindText('baseFont', ui.baseFont, config); + settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config); settings.addHTML('line2', '
'); settings.hideTitle('line2'); settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val); settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val); @@ -234,18 +260,21 @@ function setupGUI() { }); } -async function setupCanvas() { - const video = document.getElementById('video'); +async function setupCanvas(input) { + // setup canvas object to same size as input as camera resolution may change const canvas = document.getElementById('canvas'); - canvas.width = video.width; - canvas.height = video.height; + canvas.width = input.width; + canvas.height = input.height; + return canvas; } +// eslint-disable-next-line no-unused-vars async function setupCamera() { + // setup webcam. note that navigator.mediaDevices requires that page is accessed via https const video = document.getElementById('video'); if (!navigator.mediaDevices) { document.getElementById('log').innerText = 'Video not supported'; - return; + return null; } const stream = await navigator.mediaDevices.getUserMedia({ audio: false, @@ -263,27 +292,30 @@ async function setupCamera() { } // eslint-disable-next-line no-unused-vars -async function runImageDemo() { +async function setupImage() { const image = document.getElementById('image'); - const canvas = document.getElementById('canvas'); - canvas.width = image.width; - canvas.height = image.height; - const result = await human.detect(image, config); - const ctx = canvas.getContext('2d'); - ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, canvas.width, canvas.height); - drawFace(result.face); - drawBody(result.body); - drawHand(result.hand); + image.width = window.innerWidth; + image.height = window.innerHeight; + return new Promise((resolve) => { + image.onload = () => resolve(image); + image.src = 'sample.jpg'; + }); } async function main() { + // initialize tensorflow await human.tf.setBackend('webgl'); await human.tf.ready(); - await setupCamera(); - await setupCanvas(); + // setup ui control panel await setupGUI(); - runHumanDetect(); - // runImageDemo(); + // setup webcam + const video = await setupCamera(); + // or setup image + // const image = await setupImage(); + // setup output canvas from input object, select video or image + const canvas = await setupCanvas(video); + // run actual detection. if input is video, it will run in a loop else it will run only once + runHumanDetect(video, canvas); } window.onload = main; diff --git a/demo/demo-iife.js b/demo/demo-iife.js index 9534639d..8887a5c2 100644 --- a/demo/demo-iife.js +++ b/demo/demo-iife.js @@ -1,4 +1,3 @@ -/* eslint-disable no-return-assign */ /* global human, QuickSettings */ const config = { diff --git a/src/handpose/hand.js b/src/handpose/hand.js index 8194bf55..0c790f0e 100644 --- a/src/handpose/hand.js +++ b/src/handpose/hand.js @@ -92,12 +92,10 @@ class HandDetector { image.dispose(); return null; } - // Calling arraySync on both boxes and palmLandmarks because the tensors are - // very small so it's not worth calling await array(). - const boundingBoxes = prediction.boxes.arraySync(); + const boundingBoxes = await prediction.boxes.array(); const startPoint = boundingBoxes[0].slice(0, 2); const endPoint = boundingBoxes[0].slice(2, 4); - const palmLandmarks = prediction.palmLandmarks.arraySync(); + const palmLandmarks = await prediction.palmLandmarks.array(); image.dispose(); prediction.boxes.dispose(); prediction.palmLandmarks.dispose(); diff --git a/src/handpose/index.js b/src/handpose/index.js index 0be8a58c..b4f42ad7 100644 --- a/src/handpose/index.js +++ b/src/handpose/index.js @@ -56,18 +56,6 @@ class HandPose { this.pipeline = pipeline; } - static getAnnotations() { - return keypoints.MESH_ANNOTATIONS; - } - - /** - * Finds hands in the input image. - * - * @param input The image to classify. Can be a tensor, DOM element image, - * video, or canvas. - * @param flipHorizontal Whether to flip the hand keypoints horizontally. - * Should be true for videos that are flipped by default (e.g. webcams). - */ async estimateHands(input, config) { const image = tf.tidy(() => { if (!(input instanceof tf.Tensor)) {