ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

updated demo

pull/50/head
Vladimir Mandic 2020-10-13 09:59:21 -04:00
parent b02e4971da
commit 08f8c6af15
6 changed files with 90 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintrc.json
View File

@ -39,6 +39,7 @@
"no-restricted-globals": "off", "no-restricted-globals": "off",
"no-restricted-syntax": "off", "no-restricted-syntax": "off",
"no-underscore-dangle": "off", "no-underscore-dangle": "off",
"no-return-assign": "off",
"newline-per-chained-call": "off", "newline-per-chained-call": "off",
"node/no-unsupported-features/es-syntax": "off", "node/no-unsupported-features/es-syntax": "off",
"node/shebang": "off", "node/shebang": "off",

2
demo/demo-esm.html
View File

@ -1,5 +1,5 @@
<head> <head>
<script src="https://cdn.jsdelivr.net/npm/quicksettings@latest/quicksettings.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/quicksettings"></script>
<script src="./demo-esm.js" type="module"></script> <script src="./demo-esm.js" type="module"></script>
</head> </head>
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'"> <body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'">

136
demo/demo-esm.js
View File

@ -1,8 +1,13 @@
/* global QuickSettings */ /* global QuickSettings */
/* eslint-disable no-return-assign */
import human from '../dist/human.esm.js'; import human from '../dist/human.esm.js';
const ui = {
baseColor: 'rgba(255, 200, 255, 0.3)',
baseFont: 'small-caps 1.2rem "Segoe UI"',
baseLineWidth: 16,
};
const config = { const config = {
face: { face: {
enabled: true, enabled: true,
@ -17,17 +22,20 @@ const config = {
}; };
let settings; let settings;
async function drawFace(result) { async function drawFace(result, canvas) {
const canvas = document.getElementById('canvas');
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.fillStyle = 'lightcoral'; ctx.fillStyle = ui.baseColor;
ctx.strokeStyle = 'lightcoral'; ctx.strokeStyle = ui.baseColor;
ctx.font = 'small-caps 1rem "Segoe UI"'; ctx.font = ui.baseFont;
for (const face of result) { for (const face of result) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath(); ctx.beginPath();
if (settings.getValue('Draw Boxes')) {
ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]); ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 16, face.box[2]); }
ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 22, face.box[2]);
ctx.stroke(); ctx.stroke();
ctx.lineWidth = 1;
if (face.mesh) { if (face.mesh) {
if (settings.getValue('Draw Points')) { if (settings.getValue('Draw Points')) {
for (const point of face.mesh) { for (const point of face.mesh) {
@ -50,10 +58,10 @@ async function drawFace(result) {
path.lineTo(point[0], point[1]); path.lineTo(point[0], point[1]);
} }
path.closePath(); path.closePath();
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.5)`; ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
ctx.strokeStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.5)`;
ctx.stroke(path); ctx.stroke(path);
if (settings.getValue('Fill Polygons')) { if (settings.getValue('Fill Polygons')) {
ctx.fillStyle = `rgba(${127.5 + (2 * points[0][2])}, ${127.5 - (2 * points[0][2])}, 255, 0.3)`;
ctx.fill(path); ctx.fill(path);
} }
} }
@ -62,12 +70,12 @@ async function drawFace(result) {
} }
} }
async function drawBody(result) { async function drawBody(result, canvas) {
const canvas = document.getElementById('canvas');
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.fillStyle = 'lightcoral'; ctx.fillStyle = ui.baseColor;
ctx.strokeStyle = 'lightcoral'; ctx.strokeStyle = ui.baseColor;
ctx.font = 'small-caps 1rem "Segoe UI"'; ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
for (const pose of result) { for (const pose of result) {
if (settings.getValue('Draw Points')) { if (settings.getValue('Draw Points')) {
for (const point of pose.keypoints) { for (const point of pose.keypoints) {
@ -123,12 +131,19 @@ async function drawBody(result) {
} }
} }
async function drawHand(result) { async function drawHand(result, canvas) {
const canvas = document.getElementById('canvas');
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.font = 'small-caps 1rem "Segoe UI"'; ctx.font = ui.baseFont;
ctx.lineWidth = ui.baseLineWidth;
window.result = result; window.result = result;
for (const hand of result) { for (const hand of result) {
if (settings.getValue('Draw Boxes')) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath();
ctx.rect(hand.box[0], hand.box[1], hand.box[2], hand.box[3]);
ctx.fillText('hand', hand.box[0] + 2, hand.box[1] + 22, hand.box[2]);
ctx.stroke();
}
if (settings.getValue('Draw Points')) { if (settings.getValue('Draw Points')) {
for (const point of hand.landmarks) { for (const point of hand.landmarks) {
ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`; ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
@ -139,13 +154,14 @@ async function drawHand(result) {
} }
if (settings.getValue('Draw Polygons')) { if (settings.getValue('Draw Polygons')) {
const addPart = (part) => { const addPart = (part) => {
for (let i = 1; i < part.length; i++) {
ctx.lineWidth = ui.baseLineWidth;
ctx.beginPath(); ctx.beginPath();
for (const i in part) {
ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`; ctx.strokeStyle = `rgba(${127.5 + (2 * part[i][2])}, ${127.5 - (2 * part[i][2])}, 255, 0.5)`;
if (i === 0) ctx.moveTo(part[i][0], part[i][1]); ctx.moveTo(part[i - 1][0], part[i - 1][1]);
else ctx.lineTo(part[i][0], part[i][1]); ctx.lineTo(part[i][0], part[i][1]);
}
ctx.stroke(); ctx.stroke();
}
}; };
addPart(hand.annotations.indexFinger); addPart(hand.annotations.indexFinger);
addPart(hand.annotations.middleFinger); addPart(hand.annotations.middleFinger);
@ -157,25 +173,30 @@ async function drawHand(result) {
} }
} }
async function runHumanDetect() { async function runHumanDetect(input, canvas) {
const video = document.getElementById('video');
const canvas = document.getElementById('canvas');
const log = document.getElementById('log'); const log = document.getElementById('log');
const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false; const live = input.srcObject ? ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused)) : false;
if (live) { // perform detect if live video or not video at all
if (live || !(input instanceof HTMLVideoElement)) {
// perform detection // perform detection
const t0 = performance.now(); const t0 = performance.now();
const result = await human.detect(video, config); let result;
try {
result = await human.detect(input, config);
} catch (err) {
log.innerText = err.message;
}
if (!result) return;
const t1 = performance.now(); const t1 = performance.now();
// update fps // update fps
settings.setValue('FPS', Math.round(1000 / (t1 - t0))); settings.setValue('FPS', Math.round(1000 / (t1 - t0)));
// draw image from video // draw image from video
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
ctx.drawImage(video, 0, 0, video.width, video.height, 0, 0, canvas.width, canvas.height); ctx.drawImage(input, 0, 0, input.width, input.height, 0, 0, canvas.width, canvas.height);
// draw all results // draw all results
drawFace(result.face); drawFace(result.face, canvas);
drawBody(result.body); drawBody(result.body, canvas);
drawHand(result.hand); drawHand(result.hand, canvas);
// update log // update log
const engine = await human.tf.engine(); const engine = await human.tf.engine();
log.innerText = ` log.innerText = `
@ -184,12 +205,13 @@ async function runHumanDetect() {
Result Object Size: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes Result Object Size: Face: ${(JSON.stringify(result.face)).length.toLocaleString()} bytes Body: ${(JSON.stringify(result.body)).length.toLocaleString()} bytes Hand: ${(JSON.stringify(result.hand)).length.toLocaleString()} bytes
`; `;
// rinse & repeate // rinse & repeate
// setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes // if (input.readyState) setTimeout(() => runHumanDetect(), 1000); // slow loop for debugging purposes
requestAnimationFrame(runHumanDetect); // immediate loop if (input.readyState) requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
} }
} }
function setupGUI() { function setupGUI() {
// add all variables to ui control panel
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
settings.addRange('FPS', 0, 100, 0, 1); settings.addRange('FPS', 0, 100, 0, 1);
settings.addBoolean('Pause', false, (val) => { settings.addBoolean('Pause', false, (val) => {
@ -198,9 +220,13 @@ function setupGUI() {
runHumanDetect(); runHumanDetect();
}); });
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1'); settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
settings.addBoolean('Draw Boxes', false);
settings.addBoolean('Draw Points', true); settings.addBoolean('Draw Points', true);
settings.addBoolean('Draw Polygons', true); settings.addBoolean('Draw Polygons', true);
settings.addBoolean('Fill Polygons', true); settings.addBoolean('Fill Polygons', true);
settings.bindText('baseColor', ui.baseColor, config);
settings.bindText('baseFont', ui.baseFont, config);
settings.bindRange('baseLineWidth', 1, 100, ui.baseLineWidth, 1, config);
settings.addHTML('line2', '<hr>'); settings.hideTitle('line2'); settings.addHTML('line2', '<hr>'); settings.hideTitle('line2');
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val); settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val); settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
@ -234,18 +260,21 @@ function setupGUI() {
}); });
} }
async function setupCanvas() { async function setupCanvas(input) {
const video = document.getElementById('video'); // setup canvas object to same size as input as camera resolution may change
const canvas = document.getElementById('canvas'); const canvas = document.getElementById('canvas');
canvas.width = video.width; canvas.width = input.width;
canvas.height = video.height; canvas.height = input.height;
return canvas;
} }
// eslint-disable-next-line no-unused-vars
async function setupCamera() { async function setupCamera() {
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
const video = document.getElementById('video'); const video = document.getElementById('video');
if (!navigator.mediaDevices) { if (!navigator.mediaDevices) {
document.getElementById('log').innerText = 'Video not supported'; document.getElementById('log').innerText = 'Video not supported';
return; return null;
} }
const stream = await navigator.mediaDevices.getUserMedia({ const stream = await navigator.mediaDevices.getUserMedia({
audio: false, audio: false,
@ -263,27 +292,30 @@ async function setupCamera() {
} }
// eslint-disable-next-line no-unused-vars // eslint-disable-next-line no-unused-vars
async function runImageDemo() { async function setupImage() {
const image = document.getElementById('image'); const image = document.getElementById('image');
const canvas = document.getElementById('canvas'); image.width = window.innerWidth;
canvas.width = image.width; image.height = window.innerHeight;
canvas.height = image.height; return new Promise((resolve) => {
const result = await human.detect(image, config); image.onload = () => resolve(image);
const ctx = canvas.getContext('2d'); image.src = 'sample.jpg';
ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, canvas.width, canvas.height); });
drawFace(result.face);
drawBody(result.body);
drawHand(result.hand);
} }
async function main() { async function main() {
// initialize tensorflow
await human.tf.setBackend('webgl'); await human.tf.setBackend('webgl');
await human.tf.ready(); await human.tf.ready();
await setupCamera(); // setup ui control panel
await setupCanvas();
await setupGUI(); await setupGUI();
runHumanDetect(); // setup webcam
// runImageDemo(); const video = await setupCamera();
// or setup image
// const image = await setupImage();
// setup output canvas from input object, select video or image
const canvas = await setupCanvas(video);
// run actual detection. if input is video, it will run in a loop else it will run only once
runHumanDetect(video, canvas);
} }
window.onload = main; window.onload = main;

1
demo/demo-iife.js
View File

@ -1,4 +1,3 @@
/* eslint-disable no-return-assign */
/* global human, QuickSettings */ /* global human, QuickSettings */
const config = { const config = {

6
src/handpose/hand.js
View File

@ -92,12 +92,10 @@ class HandDetector {
image.dispose(); image.dispose();
return null; return null;
} }
// Calling arraySync on both boxes and palmLandmarks because the tensors are const boundingBoxes = await prediction.boxes.array();
// very small so it's not worth calling await array().
const boundingBoxes = prediction.boxes.arraySync();
const startPoint = boundingBoxes[0].slice(0, 2); const startPoint = boundingBoxes[0].slice(0, 2);
const endPoint = boundingBoxes[0].slice(2, 4); const endPoint = boundingBoxes[0].slice(2, 4);
const palmLandmarks = prediction.palmLandmarks.arraySync(); const palmLandmarks = await prediction.palmLandmarks.array();
image.dispose(); image.dispose();
prediction.boxes.dispose(); prediction.boxes.dispose();
prediction.palmLandmarks.dispose(); prediction.palmLandmarks.dispose();

12
src/handpose/index.js
View File

@ -56,18 +56,6 @@ class HandPose {
this.pipeline = pipeline; this.pipeline = pipeline;
} }
static getAnnotations() {
return keypoints.MESH_ANNOTATIONS;
}
/**
* Finds hands in the input image.
*
* @param input The image to classify. Can be a tensor, DOM element image,
* video, or canvas.
* @param flipHorizontal Whether to flip the hand keypoints horizontally.
* Should be true for videos that are flipped by default (e.g. webcams).
*/
async estimateHands(input, config) { async estimateHands(input, config) {
const image = tf.tidy(() => { const image = tf.tidy(() => {
if (!(input instanceof tf.Tensor)) { if (!(input instanceof tf.Tensor)) {