new menu layout

pull/280/head
Vladimir Mandic 2020-10-17 20:59:43 -04:00
parent 12c5350174
commit 2f60e4c72c
3 changed files with 237 additions and 89 deletions

View File

@ -1,31 +1,32 @@
/* global QuickSettings */
import human from '../dist/human.esm.js';
import draw from './draw.js';
import Menu from './menu.js';
// ui options
const ui = {
baseColor: 'rgba(255, 200, 255, 0.3)',
baseLabel: 'rgba(255, 200, 255, 0.9)',
baseColor: 'rgba(173, 216, 230, 0.3)', // this is 'lightblue', just with alpha channel
baseLabel: 'rgba(173, 216, 230, 0.9)',
baseFontProto: 'small-caps {size} "Segoe UI"',
baseLineWidth: 16,
baseLineHeightProto: 2,
columns: 3,
columns: 2,
busy: false,
facing: 'user',
facing: true,
useWorker: false,
worker: 'worker.js',
samples: ['../assets/sample1.jpg', '../assets/sample2.jpg', '../assets/sample3.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample6.jpg'],
samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'],
drawBoxes: true,
drawPoints: false,
drawPolygons: true,
fillPolygons: true,
useDepth: true,
console: true,
maxFrames: 10,
};
// configuration overrides
const config = {
backend: 'webgl', // if you want to use 'wasm' backend, enable script load of tf and tf-backend-wasm in index.html
face: {
enabled: true,
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
@ -40,7 +41,7 @@ const config = {
};
// global variables
let settings;
let menu;
let worker;
let timeStamp;
const fps = [];
@ -63,12 +64,11 @@ const log = (...msg) => {
};
// draws processed results and starts processing of a next frame
async function drawResults(input, result, canvas) {
function drawResults(input, result, canvas) {
// update fps
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
fps.push(1000 / (performance.now() - timeStamp));
if (fps.length > 20) fps.shift();
settings.setValue('FPS', Math.round(10 * fps.reduce((a, b) => a + b) / fps.length) / 10);
if (fps.length > ui.maxFrames) fps.shift();
menu.updateChart('FPS', fps);
// eslint-disable-next-line no-use-before-define
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
@ -81,7 +81,7 @@ async function drawResults(input, result, canvas) {
draw.body(result.body, canvas, ui);
draw.hand(result.hand, canvas, ui);
// update log
const engine = await human.tf.engine();
const engine = human.tf.engine();
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
const gpu = engine.backendInstance ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
document.getElementById('log').innerText = `
@ -98,7 +98,7 @@ async function setupCamera() {
const canvas = document.getElementById('canvas');
const output = document.getElementById('log');
const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false;
let msg = `Setting up camera: live: ${live} facing: ${ui.facing}`;
let msg = `Setting up camera: live: ${live} facing: ${ui.facing ? 'front' : 'back'}`;
output.innerText += `\n${msg}`;
log(msg);
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
@ -112,7 +112,7 @@ async function setupCamera() {
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: { facingMode: ui.facing, width: window.innerWidth, height: window.innerHeight },
video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight },
});
} catch (err) {
output.innerText += '\nCamera permission denied';
@ -150,7 +150,7 @@ function webWorker(input, image, canvas) {
}
// main processing function when input is webcam, can use direct invocation or web worker
async function runHumanDetect(input, canvas) {
function runHumanDetect(input, canvas) {
timeStamp = performance.now();
// perform detect if live video or not video at all
if (input.srcObject) {
@ -170,36 +170,23 @@ async function runHumanDetect(input, canvas) {
// perform detection in worker
webWorker(input, data, canvas);
} else {
let result = {};
try {
// perform detection
result = await human.detect(input, config);
} catch (err) {
log('Error during execution:', err.message);
}
if (result.error) log(result.error);
else drawResults(input, result, canvas);
human.detect(input, config).then((result) => {
if (result.error) log(result.error);
else drawResults(input, result, canvas);
});
}
}
}
// main processing function when input is image, can use direct invocation or web worker
async function processImage(input) {
const cfg = {
backend: 'webgl',
console: true,
face: {
enabled: true,
detector: { maxFaces: 10, skipFrames: 0, minConfidence: 0.1, iouThreshold: 0.3, scoreThreshold: 0.3 },
mesh: { enabled: true },
iris: { enabled: true },
age: { enabled: true, skipFrames: 0 },
gender: { enabled: true },
emotion: { enabled: true, minConfidence: 0.1, useGrayscale: true },
},
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
hand: { enabled: true, skipFrames: 0, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.5 },
};
// must be zero for images
config.face.detector.skipFrames = 0;
config.face.emotion.skipFrames = 0;
config.face.age.skipFrames = 0;
config.hand.skipFrames = 0;
timeStamp = performance.now();
return new Promise((resolve) => {
const image = document.getElementById('image');
image.onload = async () => {
@ -209,11 +196,13 @@ async function processImage(input) {
image.height = image.naturalHeight;
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
const result = await human.detect(image, cfg);
await drawResults(image, result, canvas);
const result = await human.detect(image, config);
drawResults(image, result, canvas);
const thumb = document.createElement('canvas');
thumb.width = window.innerWidth / (ui.columns + 0.02);
thumb.width = (window.innerWidth - menu.width) / (ui.columns + 0.1);
thumb.height = canvas.height / (window.innerWidth / thumb.width);
thumb.style.margin = '8px';
thumb.style.boxShadow = '4px 4px 4px 0 dimgrey';
const ctx = thumb.getContext('2d');
ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, thumb.width, thumb.height);
document.getElementById('samples').appendChild(thumb);
@ -253,74 +242,68 @@ async function detectSampleImages() {
for (const sample of ui.samples) await processImage(sample);
}
// setup settings panel
function setupUI() {
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
const style = document.createElement('style');
style.innerHTML = `
.qs_main { font: 1rem "Segoe UI"; }
.qs_label { font: 0.8rem "Segoe UI"; }
.qs_content { background: darkslategray; }
.qs_container { background: transparent; color: white; margin: 6px; padding: 6px; }
.qs_checkbox_label { top: 2px; }
.qs_button { width: -webkit-fill-available; font: 1rem "Segoe UI"; cursor: pointer; }
`;
document.getElementsByTagName('head')[0].appendChild(style);
settings.addButton('Play/Pause WebCam', () => detectVideo());
settings.addButton('Process Images', () => detectSampleImages());
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => config.backend = val.value);
settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title');
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
settings.addBoolean('Face Emotion', config.face.emotion.enabled, (val) => config.face.emotion.enabled = val);
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
settings.addHTML('title', 'Model Parameters'); settings.hideTitle('title');
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
function setupMenu() {
menu = new Menu(document.body);
menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt));
menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Enabled Models');
menu.addBool('Face Detect', config.face, 'enabled');
menu.addBool('Face Mesh', config.face.mesh, 'enabled');
menu.addBool('Face Iris', config.face.iris, 'enabled');
menu.addBool('Face Age', config.face.age, 'enabled');
menu.addBool('Face Gender', config.face.gender, 'enabled');
menu.addBool('Face Emotion', config.face.emotion, 'enabled');
menu.addBool('Body Pose', config.body, 'enabled');
menu.addBool('Hand Pose', config.hand, 'enabled');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Model Parameters');
menu.addRange('Max Objects', config.face.detector, 'maxFaces', 0, 50, 1, (val) => {
config.face.detector.maxFaces = parseInt(val);
config.body.maxDetections = parseInt(val);
config.hand.maxHands = parseInt(val);
});
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
menu.addRange('Skip Frames', config.face.detector, 'skipFrames', 0, 50, 1, (val) => {
config.face.detector.skipFrames = parseInt(val);
config.face.emotion.skipFrames = parseInt(val);
config.face.age.skipFrames = parseInt(val);
config.hand.skipFrames = parseInt(val);
});
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
menu.addRange('Min Confidence', config.face.detector, 'minConfidence', 0.0, 1.0, 0.05, (val) => {
config.face.detector.minConfidence = parseFloat(val);
config.face.emotion.minConfidence = parseFloat(val);
config.hand.minConfidence = parseFloat(val);
});
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
menu.addRange('Score Threshold', config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => {
config.face.detector.scoreThreshold = parseFloat(val);
config.hand.scoreThreshold = parseFloat(val);
config.body.scoreThreshold = parseFloat(val);
});
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
menu.addRange('IOU Threshold', config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => {
config.face.detector.iouThreshold = parseFloat(val);
config.hand.iouThreshold = parseFloat(val);
});
settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
settings.addBoolean('Use Web Worker', ui.useWorker, (val) => ui.useWorker = val);
settings.addBoolean('Camera Front/Back', true, (val) => {
ui.facing = val ? 'user' : 'environment';
setupCamera();
});
settings.addBoolean('Use 3D Depth', ui.useDepth, (val) => ui.useDepth = val);
settings.addBoolean('Draw Boxes', ui.drawBoxes, (val) => ui.drawBoxes = val);
settings.addBoolean('Draw Points', ui.drawPoints, (val) => ui.drawPoints = val);
settings.addBoolean('Draw Polygons', ui.drawPolygons, (val) => ui.drawPolygons = val);
settings.addBoolean('Fill Polygons', ui.fillPolygons, (val) => ui.fillPolygons = val);
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
settings.addRange('FPS', 0, 100, 0, 1);
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('UI Options');
menu.addBool('Use Web Worker', ui, 'useWorker');
menu.addBool('Camera Front/Back', ui, 'facing', () => setupCamera());
menu.addBool('Use 3D Depth', ui, 'useDepth');
menu.addBool('Draw Boxes', ui, 'drawBoxes');
menu.addBool('Draw Points', ui, 'drawPoints');
menu.addBool('Draw Polygons', ui, 'drawPolygons');
menu.addBool('Fill Polygons', ui, 'fillPolygons');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addValue('State', '');
menu.addChart('FPS', 'FPS');
}
async function main() {
log('Human demo starting ...');
setupUI();
setupMenu();
const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`;
document.getElementById('log').innerText += '\n' + msg;
log(msg);

View File

@ -14,7 +14,6 @@
<link rel="shortcut icon" href="../favicon.ico" type="image/x-icon">
<!-- <script src="../assets/tf.min.js"></script> -->
<!-- <script src="../assets/tf-backend-wasm.min.js"></script> -->
<script src="../assets/quicksettings.js"></script>
<script src="./browser.js" type="module"></script>
</head>
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; overflow-x: hidden">

166
demo/menu.js Normal file
View File

@ -0,0 +1,166 @@
const css = `
.menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); }
.menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; }
.menu-title { padding: 0; }
.menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) }
.menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
.menu-chart-title { align-items: center; }
.menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; }
.menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; }
.menu-button:hover { background: lightgreen; }
.menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; }
.menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
.menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
input[type=checkbox] { visibility: hidden; }
input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; }
.menu-range { margin: 0 0.8rem 0 0; width: 5rem; background: transparent; color: lightblue; }
.menu-range:before { content: attr(value); color: white; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; }
input[type=range] { -webkit-appearance: none; }
input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: black; border-radius: 1rem; border: 1px; }
input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: 1rem; background: lightblue; cursor: pointer; -webkit-appearance: none; }
`;
function createCSS() {
const el = document.createElement('style');
el.innerHTML = css;
document.getElementsByTagName('head')[0].appendChild(el);
}
function createElem(parent) {
const el = document.createElement('div');
el.id = 'menu';
el.className = 'menu-container';
if (typeof parent === 'object') parent.appendChild(el);
else document.getElementById(parent).appendChild(el);
return el;
}
class Menu {
constructor(parent) {
createCSS();
this.menu = createElem(parent);
this._id = 0;
this._maxFPS = 0;
}
get newID() {
this._id++;
return `menu-${this._id}`;
}
get ID() {
return `menu-${this._id}`;
}
get width() {
return this.menu.offsetWidth;
}
get height() {
return this.menu.offsetHeight;
}
async addLabel(title) {
const el = document.createElement('div');
el.className = 'menu menu-title';
el.id = this.newID;
el.innerHTML = title;
this.menu.appendChild(el);
}
async addBool(title, object, variable, callback) {
const el = document.createElement('div');
el.className = 'menu';
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-label" for="${this.ID}"></label></div>${title}`;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('change', (evt) => {
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
});
}
async addRange(title, object, variable, min, max, step, callback) {
const el = document.createElement('div');
el.className = 'menu';
el.innerHTML = `<input class="menu-range" type="range" id="${this.newID}" min="${min}" max="${max}" step="${step}" value="${object[variable]}">${title}`;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('change', (evt) => {
object[variable] = evt.target.value;
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target.value);
});
}
async addHTML(html) {
const el = document.createElement('div');
el.className = 'menu';
el.id = this.newID;
if (html) el.innerHTML = html;
this.menu.appendChild(el);
}
async addButton(titleOn, titleOff, callback) {
const el = document.createElement('button');
el.className = 'menu menu-button';
el.type = 'button';
el.id = this.newID;
el.innerText = titleOn;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('click', () => {
if (el.innerText === titleOn) el.innerText = titleOff;
else el.innerText = titleOn;
if (callback) callback(el.innerText !== titleOn);
});
}
async addValue(title, val) {
const el = document.createElement('div');
el.className = 'menu';
el.id = title;
el.innerText = `${title}: ${val}`;
this.menu.appendChild(el);
}
// eslint-disable-next-line class-methods-use-this
async updateValue(title, val) {
const el = document.getElementById(title);
el.innerText = `${title}: ${val}`;
}
async addChart(title, id) {
const el = document.createElement('div');
el.className = 'menu menu-chart-title';
el.id = this.newID;
el.innerHTML = `${title}<canvas id="menu-canvas-${id}" class="menu-chart-canvas" width="180px" height="40px"></canvas>`;
this.menu.appendChild(el);
}
// eslint-disable-next-line class-methods-use-this
async updateChart(id, values) {
if (!values || (values.length === 0)) return;
const canvas = document.getElementById(`menu-canvas-${id}`);
if (!canvas) return;
const ctx = canvas.getContext('2d');
ctx.fillStyle = 'darkslategray';
ctx.fillRect(0, 0, canvas.width, canvas.height);
const width = canvas.width / values.length;
const max = 1 + Math.max(...values);
const height = canvas.height / max;
for (const i in values) {
const gradient = ctx.createLinearGradient(0, (max - values[i]) * height, 0, 0);
gradient.addColorStop(0.1, 'lightblue');
gradient.addColorStop(0.4, 'darkslategray');
ctx.fillStyle = gradient;
ctx.fillRect(i * width, 0, width - 4, canvas.height);
ctx.fillStyle = 'black';
ctx.font = '12px "Segoe UI"';
ctx.fillText(Math.round(values[i]), i * width, canvas.height - 2, width);
}
}
}
export default Menu;