new menu layout

pull/50/head
Vladimir Mandic 2020-10-17 20:59:43 -04:00
parent 90b192c890
commit a52ba56e70
5 changed files with 237 additions and 82661 deletions

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@ -1,31 +1,32 @@
/* global QuickSettings */
import human from '../dist/human.esm.js';
import draw from './draw.js';
import Menu from './menu.js';
// ui options
const ui = {
baseColor: 'rgba(255, 200, 255, 0.3)',
baseLabel: 'rgba(255, 200, 255, 0.9)',
baseColor: 'rgba(173, 216, 230, 0.3)', // this is 'lightblue', just with alpha channel
baseLabel: 'rgba(173, 216, 230, 0.9)',
baseFontProto: 'small-caps {size} "Segoe UI"',
baseLineWidth: 16,
baseLineHeightProto: 2,
columns: 3,
columns: 2,
busy: false,
facing: 'user',
facing: true,
useWorker: false,
worker: 'worker.js',
samples: ['../assets/sample1.jpg', '../assets/sample2.jpg', '../assets/sample3.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample6.jpg'],
samples: ['../assets/sample6.jpg', '../assets/sample1.jpg', '../assets/sample4.jpg', '../assets/sample5.jpg', '../assets/sample3.jpg', '../assets/sample2.jpg'],
drawBoxes: true,
drawPoints: false,
drawPolygons: true,
fillPolygons: true,
useDepth: true,
console: true,
maxFrames: 10,
};
// configuration overrides
const config = {
backend: 'webgl', // if you want to use 'wasm' backend, enable script load of tf and tf-backend-wasm in index.html
face: {
enabled: true,
detector: { maxFaces: 10, skipFrames: 10, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.7 },
@ -40,7 +41,7 @@ const config = {
};
// global variables
let settings;
let menu;
let worker;
let timeStamp;
const fps = [];
@ -63,12 +64,11 @@ const log = (...msg) => {
};
// draws processed results and starts processing of a next frame
async function drawResults(input, result, canvas) {
function drawResults(input, result, canvas) {
// update fps
settings.setValue('FPS', Math.round(1000 / (performance.now() - timeStamp)));
fps.push(1000 / (performance.now() - timeStamp));
if (fps.length > 20) fps.shift();
settings.setValue('FPS', Math.round(10 * fps.reduce((a, b) => a + b) / fps.length) / 10);
if (fps.length > ui.maxFrames) fps.shift();
menu.updateChart('FPS', fps);
// eslint-disable-next-line no-use-before-define
requestAnimationFrame(() => runHumanDetect(input, canvas)); // immediate loop
@ -81,7 +81,7 @@ async function drawResults(input, result, canvas) {
draw.body(result.body, canvas, ui);
draw.hand(result.hand, canvas, ui);
// update log
const engine = await human.tf.engine();
const engine = human.tf.engine();
const memory = `${engine.state.numBytes.toLocaleString()} bytes ${engine.state.numDataBuffers.toLocaleString()} buffers ${engine.state.numTensors.toLocaleString()} tensors`;
const gpu = engine.backendInstance ? `GPU: ${engine.backendInstance.numBytesInGPU.toLocaleString()} bytes` : '';
document.getElementById('log').innerText = `
@ -98,7 +98,7 @@ async function setupCamera() {
const canvas = document.getElementById('canvas');
const output = document.getElementById('log');
const live = video.srcObject ? ((video.srcObject.getVideoTracks()[0].readyState === 'live') && (video.readyState > 2) && (!video.paused)) : false;
let msg = `Setting up camera: live: ${live} facing: ${ui.facing}`;
let msg = `Setting up camera: live: ${live} facing: ${ui.facing ? 'front' : 'back'}`;
output.innerText += `\n${msg}`;
log(msg);
// setup webcam. note that navigator.mediaDevices requires that page is accessed via https
@ -112,7 +112,7 @@ async function setupCamera() {
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: { facingMode: ui.facing, width: window.innerWidth, height: window.innerHeight },
video: { facingMode: (ui.facing ? 'user' : 'environment'), width: window.innerWidth, height: window.innerHeight },
});
} catch (err) {
output.innerText += '\nCamera permission denied';
@ -150,7 +150,7 @@ function webWorker(input, image, canvas) {
}
// main processing function when input is webcam, can use direct invocation or web worker
async function runHumanDetect(input, canvas) {
function runHumanDetect(input, canvas) {
timeStamp = performance.now();
// perform detect if live video or not video at all
if (input.srcObject) {
@ -170,36 +170,23 @@ async function runHumanDetect(input, canvas) {
// perform detection in worker
webWorker(input, data, canvas);
} else {
let result = {};
try {
// perform detection
result = await human.detect(input, config);
} catch (err) {
log('Error during execution:', err.message);
}
human.detect(input, config).then((result) => {
if (result.error) log(result.error);
else drawResults(input, result, canvas);
});
}
}
}
// main processing function when input is image, can use direct invocation or web worker
async function processImage(input) {
const cfg = {
backend: 'webgl',
console: true,
face: {
enabled: true,
detector: { maxFaces: 10, skipFrames: 0, minConfidence: 0.1, iouThreshold: 0.3, scoreThreshold: 0.3 },
mesh: { enabled: true },
iris: { enabled: true },
age: { enabled: true, skipFrames: 0 },
gender: { enabled: true },
emotion: { enabled: true, minConfidence: 0.1, useGrayscale: true },
},
body: { enabled: true, maxDetections: 10, scoreThreshold: 0.7, nmsRadius: 20 },
hand: { enabled: true, skipFrames: 0, minConfidence: 0.5, iouThreshold: 0.3, scoreThreshold: 0.5 },
};
// must be zero for images
config.face.detector.skipFrames = 0;
config.face.emotion.skipFrames = 0;
config.face.age.skipFrames = 0;
config.hand.skipFrames = 0;
timeStamp = performance.now();
return new Promise((resolve) => {
const image = document.getElementById('image');
image.onload = async () => {
@ -209,11 +196,13 @@ async function processImage(input) {
image.height = image.naturalHeight;
canvas.width = image.naturalWidth;
canvas.height = image.naturalHeight;
const result = await human.detect(image, cfg);
await drawResults(image, result, canvas);
const result = await human.detect(image, config);
drawResults(image, result, canvas);
const thumb = document.createElement('canvas');
thumb.width = window.innerWidth / (ui.columns + 0.02);
thumb.width = (window.innerWidth - menu.width) / (ui.columns + 0.1);
thumb.height = canvas.height / (window.innerWidth / thumb.width);
thumb.style.margin = '8px';
thumb.style.boxShadow = '4px 4px 4px 0 dimgrey';
const ctx = thumb.getContext('2d');
ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, thumb.width, thumb.height);
document.getElementById('samples').appendChild(thumb);
@ -253,74 +242,68 @@ async function detectSampleImages() {
for (const sample of ui.samples) await processImage(sample);
}
// setup settings panel
function setupUI() {
settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
const style = document.createElement('style');
style.innerHTML = `
.qs_main { font: 1rem "Segoe UI"; }
.qs_label { font: 0.8rem "Segoe UI"; }
.qs_content { background: darkslategray; }
.qs_container { background: transparent; color: white; margin: 6px; padding: 6px; }
.qs_checkbox_label { top: 2px; }
.qs_button { width: -webkit-fill-available; font: 1rem "Segoe UI"; cursor: pointer; }
`;
document.getElementsByTagName('head')[0].appendChild(style);
settings.addButton('Play/Pause WebCam', () => detectVideo());
settings.addButton('Process Images', () => detectSampleImages());
settings.addDropDown('Backend', ['webgl', 'wasm', 'cpu'], async (val) => config.backend = val.value);
settings.addHTML('title', 'Enabled Models'); settings.hideTitle('title');
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
settings.addBoolean('Face Emotion', config.face.emotion.enabled, (val) => config.face.emotion.enabled = val);
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
settings.addHTML('title', 'Model Parameters'); settings.hideTitle('title');
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
function setupMenu() {
menu = new Menu(document.body);
menu.addButton('Start Video', 'Pause Video', (evt) => detectVideo(evt));
menu.addButton('Process Images', 'Process Images', () => detectSampleImages());
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Enabled Models');
menu.addBool('Face Detect', config.face, 'enabled');
menu.addBool('Face Mesh', config.face.mesh, 'enabled');
menu.addBool('Face Iris', config.face.iris, 'enabled');
menu.addBool('Face Age', config.face.age, 'enabled');
menu.addBool('Face Gender', config.face.gender, 'enabled');
menu.addBool('Face Emotion', config.face.emotion, 'enabled');
menu.addBool('Body Pose', config.body, 'enabled');
menu.addBool('Hand Pose', config.hand, 'enabled');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('Model Parameters');
menu.addRange('Max Objects', config.face.detector, 'maxFaces', 0, 50, 1, (val) => {
config.face.detector.maxFaces = parseInt(val);
config.body.maxDetections = parseInt(val);
config.hand.maxHands = parseInt(val);
});
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
menu.addRange('Skip Frames', config.face.detector, 'skipFrames', 0, 50, 1, (val) => {
config.face.detector.skipFrames = parseInt(val);
config.face.emotion.skipFrames = parseInt(val);
config.face.age.skipFrames = parseInt(val);
config.hand.skipFrames = parseInt(val);
});
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
menu.addRange('Min Confidence', config.face.detector, 'minConfidence', 0.0, 1.0, 0.05, (val) => {
config.face.detector.minConfidence = parseFloat(val);
config.face.emotion.minConfidence = parseFloat(val);
config.hand.minConfidence = parseFloat(val);
});
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
menu.addRange('Score Threshold', config.face.detector, 'scoreThreshold', 0.1, 1.0, 0.05, (val) => {
config.face.detector.scoreThreshold = parseFloat(val);
config.hand.scoreThreshold = parseFloat(val);
config.body.scoreThreshold = parseFloat(val);
});
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
menu.addRange('IOU Threshold', config.face.detector, 'iouThreshold', 0.1, 1.0, 0.05, (val) => {
config.face.detector.iouThreshold = parseFloat(val);
config.hand.iouThreshold = parseFloat(val);
});
settings.addHTML('title', 'UI Options'); settings.hideTitle('title');
settings.addBoolean('Use Web Worker', ui.useWorker, (val) => ui.useWorker = val);
settings.addBoolean('Camera Front/Back', true, (val) => {
ui.facing = val ? 'user' : 'environment';
setupCamera();
});
settings.addBoolean('Use 3D Depth', ui.useDepth, (val) => ui.useDepth = val);
settings.addBoolean('Draw Boxes', ui.drawBoxes, (val) => ui.drawBoxes = val);
settings.addBoolean('Draw Points', ui.drawPoints, (val) => ui.drawPoints = val);
settings.addBoolean('Draw Polygons', ui.drawPolygons, (val) => ui.drawPolygons = val);
settings.addBoolean('Fill Polygons', ui.fillPolygons, (val) => ui.fillPolygons = val);
settings.addHTML('line1', '<hr>'); settings.hideTitle('line1');
settings.addRange('FPS', 0, 100, 0, 1);
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addLabel('UI Options');
menu.addBool('Use Web Worker', ui, 'useWorker');
menu.addBool('Camera Front/Back', ui, 'facing', () => setupCamera());
menu.addBool('Use 3D Depth', ui, 'useDepth');
menu.addBool('Draw Boxes', ui, 'drawBoxes');
menu.addBool('Draw Points', ui, 'drawPoints');
menu.addBool('Draw Polygons', ui, 'drawPolygons');
menu.addBool('Fill Polygons', ui, 'fillPolygons');
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addValue('State', '');
menu.addChart('FPS', 'FPS');
}
async function main() {
log('Human demo starting ...');
setupUI();
setupMenu();
const msg = `Human ready: version: ${human.version} TensorFlow/JS version: ${human.tf.version_core}`;
document.getElementById('log').innerText += '\n' + msg;
log(msg);

View File

@ -14,7 +14,6 @@
<link rel="shortcut icon" href="../favicon.ico" type="image/x-icon">
<!-- <script src="../assets/tf.min.js"></script> -->
<!-- <script src="../assets/tf-backend-wasm.min.js"></script> -->
<script src="../assets/quicksettings.js"></script>
<script src="./browser.js" type="module"></script>
</head>
<body style="margin: 0; background: black; color: white; font-family: 'Segoe UI'; font-size: 16px; font-variant: small-caps; overflow-x: hidden">

166
demo/menu.js Normal file
View File

@ -0,0 +1,166 @@
const css = `
.menu-container { display: block; background: darkslategray; position: fixed; top: 0rem; right: 0; width: fit-content; padding: 0 0.8rem 0 0.8rem; line-height: 1.8rem; z-index: 10; max-height: calc(100% - 4rem); }
.menu { display: flex; white-space: nowrap; background: darkslategray; padding: 0.2rem; width: max-content; }
.menu-title { padding: 0; }
.menu-hr { margin: 0.2rem; border: 1px solid rgba(0, 0, 0, 0.5) }
.menu-label { width: 1.3rem; height: 0.8rem; cursor: pointer; position: absolute; top: 0.1rem; left: 0.1rem; z-index: 1; background: lightcoral; border-radius: 1rem; transition: left 0.6s ease; }
.menu-chart-title { align-items: center; }
.menu-chart-canvas { background: transparent; height: 40px; width: 180px; margin: 0.2rem 0.2rem 0.2rem 1rem; }
.menu-button { border: 0; background: lightblue; width: -webkit-fill-available; padding: 8px; margin: 8px 0 8px 0; cursor: pointer; box-shadow: 4px 4px 4px 0 dimgrey; }
.menu-button:hover { background: lightgreen; }
.menu-checkbox { width: 2.8rem; height: 1rem; background: black; margin: 0.5rem 0.8rem 0 0; position: relative; border-radius: 1rem; }
.menu-checkbox:after { content: 'OFF'; color: lightcoral; position: absolute; right: 0.2rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
.menu-checkbox:before { content: 'ON'; color: lightgreen; position: absolute; left: 0.3rem; top: -0.4rem; font-weight: 800; font-size: 0.5rem; }
input[type=checkbox] { visibility: hidden; }
input[type=checkbox]:checked + label { left: 1.4rem; background: lightgreen; }
.menu-range { margin: 0 0.8rem 0 0; width: 5rem; background: transparent; color: lightblue; }
.menu-range:before { content: attr(value); color: white; margin: 0 0.4rem 0 0; font-weight: 800; font-size: 0.6rem; position: relative; top: 0.3rem; }
input[type=range] { -webkit-appearance: none; }
input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: black; border-radius: 1rem; border: 1px; }
input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: 1rem; background: lightblue; cursor: pointer; -webkit-appearance: none; }
`;
function createCSS() {
const el = document.createElement('style');
el.innerHTML = css;
document.getElementsByTagName('head')[0].appendChild(el);
}
function createElem(parent) {
const el = document.createElement('div');
el.id = 'menu';
el.className = 'menu-container';
if (typeof parent === 'object') parent.appendChild(el);
else document.getElementById(parent).appendChild(el);
return el;
}
class Menu {
constructor(parent) {
createCSS();
this.menu = createElem(parent);
this._id = 0;
this._maxFPS = 0;
}
get newID() {
this._id++;
return `menu-${this._id}`;
}
get ID() {
return `menu-${this._id}`;
}
get width() {
return this.menu.offsetWidth;
}
get height() {
return this.menu.offsetHeight;
}
async addLabel(title) {
const el = document.createElement('div');
el.className = 'menu menu-title';
el.id = this.newID;
el.innerHTML = title;
this.menu.appendChild(el);
}
async addBool(title, object, variable, callback) {
const el = document.createElement('div');
el.className = 'menu';
el.innerHTML = `<div class="menu-checkbox"><input class="menu-checkbox" type="checkbox" id="${this.newID}" ${object[variable] ? 'checked' : ''}/><label class="menu-label" for="${this.ID}"></label></div>${title}`;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('change', (evt) => {
object[variable] = evt.target.checked;
if (callback) callback(evt.target.checked);
});
}
async addRange(title, object, variable, min, max, step, callback) {
const el = document.createElement('div');
el.className = 'menu';
el.innerHTML = `<input class="menu-range" type="range" id="${this.newID}" min="${min}" max="${max}" step="${step}" value="${object[variable]}">${title}`;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('change', (evt) => {
object[variable] = evt.target.value;
evt.target.setAttribute('value', evt.target.value);
if (callback) callback(evt.target.value);
});
}
async addHTML(html) {
const el = document.createElement('div');
el.className = 'menu';
el.id = this.newID;
if (html) el.innerHTML = html;
this.menu.appendChild(el);
}
async addButton(titleOn, titleOff, callback) {
const el = document.createElement('button');
el.className = 'menu menu-button';
el.type = 'button';
el.id = this.newID;
el.innerText = titleOn;
this.menu.appendChild(el);
document.getElementById(this.ID).addEventListener('click', () => {
if (el.innerText === titleOn) el.innerText = titleOff;
else el.innerText = titleOn;
if (callback) callback(el.innerText !== titleOn);
});
}
async addValue(title, val) {
const el = document.createElement('div');
el.className = 'menu';
el.id = title;
el.innerText = `${title}: ${val}`;
this.menu.appendChild(el);
}
// eslint-disable-next-line class-methods-use-this
async updateValue(title, val) {
const el = document.getElementById(title);
el.innerText = `${title}: ${val}`;
}
async addChart(title, id) {
const el = document.createElement('div');
el.className = 'menu menu-chart-title';
el.id = this.newID;
el.innerHTML = `${title}<canvas id="menu-canvas-${id}" class="menu-chart-canvas" width="180px" height="40px"></canvas>`;
this.menu.appendChild(el);
}
// eslint-disable-next-line class-methods-use-this
async updateChart(id, values) {
if (!values || (values.length === 0)) return;
const canvas = document.getElementById(`menu-canvas-${id}`);
if (!canvas) return;
const ctx = canvas.getContext('2d');
ctx.fillStyle = 'darkslategray';
ctx.fillRect(0, 0, canvas.width, canvas.height);
const width = canvas.width / values.length;
const max = 1 + Math.max(...values);
const height = canvas.height / max;
for (const i in values) {
const gradient = ctx.createLinearGradient(0, (max - values[i]) * height, 0, 0);
gradient.addColorStop(0.1, 'lightblue');
gradient.addColorStop(0.4, 'darkslategray');
ctx.fillStyle = gradient;
ctx.fillRect(i * width, 0, width - 4, canvas.height);
ctx.fillStyle = 'black';
ctx.font = '12px "Segoe UI"';
ctx.fillText(Math.round(values[i]), i * width, canvas.height - 2, width);
}
}
}
export default Menu;