From 5516daf2d644080cefe1132497ab968d52004b7e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 11 Oct 2020 21:21:41 -0400 Subject: [PATCH] updated demo --- demo/index.html | 25 ++----- demo/index.js | 189 +++++++++++++++++++++++++----------------------- src/index.js | 6 +- 3 files changed, 109 insertions(+), 111 deletions(-) diff --git a/demo/index.html b/demo/index.html index b8481df3..891447d6 100644 --- a/demo/index.html +++ b/demo/index.html @@ -1,25 +1,12 @@ - - - + + + - +
-
-
- - -
-
-
-
+ +
- - diff --git a/demo/index.js b/demo/index.js index 30c35746..bc375e54 100644 --- a/demo/index.js +++ b/demo/index.js @@ -1,120 +1,127 @@ -/* global tf, ScatterGL, dat */ +/* eslint-disable no-return-assign */ +/* global tf, human, QuickSettings */ -import human from '../dist/human.esm.js'; - -const state = { - backend: 'webgl', - triangulateMesh: true, - renderPointcloud: true, - stop: false, - videoSize: 700, -}; -const options = { -}; - -let ctx; -let videoWidth; -let videoHeight; +let paused = false; let video; let canvas; -let scatterGLHasInitialized = false; -let scatterGL; +let ctx; -async function renderPrediction() { - const predictions = await human.detect(video); - ctx.drawImage(video, 0, 0, videoWidth, videoHeight, 0, 0, canvas.width, canvas.height); - const div = document.getElementById('faces'); - div.innerHTML = ''; - for (const prediction of predictions) { - div.appendChild(prediction.canvas); +const config = { + face: { + enabled: true, + detector: { maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 }, + mesh: { enabled: true }, + iris: { enabled: true }, + age: { enabled: false, skipFrames: 5 }, + gender: { enabled: false }, + }, + body: { enabled: false, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 }, + hand: { enabled: false, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 }, +}; + +async function drawFace(faces) { + for (const face of faces) { + ctx.drawImage(video, 0, 0, video.width, video.height, 0, 0, canvas.width, canvas.height); ctx.beginPath(); - ctx.rect(prediction.box[0], prediction.box[1], prediction.box[2], prediction.box[3]); - ctx.font = 'small-caps 1rem "Segoe UI"'; - ctx.fillText(`${prediction.gender} ${prediction.age}`, prediction.box[0] + 2, prediction.box[1] + 16, prediction.box[2]); + ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]); + ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 16, face.box[2]); ctx.stroke(); - if (state.triangulateMesh) { - for (let i = 0; i < human.triangulation.length / 3; i++) { - const points = [human.triangulation[i * 3], human.triangulation[i * 3 + 1], human.triangulation[i * 3 + 2]].map((index) => prediction.mesh[index]); - const region = new Path2D(); - region.moveTo(points[0][0], points[0][1]); - for (let j = 1; i < points.length; j++) region.lineTo(points[j][0], points[j][1]); - region.closePath(); - ctx.stroke(region); - } - } else { - for (let i = 0; i < prediction.mesh.length; i++) { - const x = prediction.mesh[i][0]; - const y = prediction.mesh[i][1]; + if (face.mesh) { + for (const point of face.mesh) { + ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`; ctx.beginPath(); - ctx.arc(x, y, 1 /* radius */, 0, 2 * Math.PI); + ctx.arc(point[0], point[1], 1 /* radius */, 0, 2 * Math.PI); ctx.fill(); } } - if (state.renderPointcloud && scatterGL != null) { - const pointsData = predictions.map((pred) => pred.mesh.map((point) => ([-point[0], -point[1], -point[2]]))); - let flattenedPointsData = []; - for (let i = 0; i < pointsData.length; i++) { - flattenedPointsData = flattenedPointsData.concat(pointsData[i]); - } - const dataset = new ScatterGL.Dataset(flattenedPointsData); - if (!scatterGLHasInitialized) scatterGL.render(dataset); - else scatterGL.updateDataset(dataset); - scatterGLHasInitialized = true; - } } - if (!state.stop) requestAnimationFrame(renderPrediction); } -function setupDatGui() { - const gui = new dat.GUI(); - gui.add(state, 'stop').onChange(() => { renderPrediction(); }); - gui.add(state, 'backend', ['webgl', 'cpu']).onChange((backend) => { tf.setBackend(backend); }); - gui.add(options, 'maxFaces', 1, 100, 1).onChange(() => { human.load(options); }); - gui.add(options, 'detectionConfidence', 0, 1, 0.05).onChange(() => { human.load(options); }); - gui.add(options, 'iouThreshold', 0, 1, 0.05).onChange(() => { human.load(options); }); - gui.add(options, 'scoreThreshold', 0, 1, 0.05).onChange(() => { human.load(options); }); - gui.add(state, 'triangulateMesh'); - gui.add(state, 'renderPointcloud').onChange((render) => { document.querySelector('#scatter-gl-container').style.display = render ? 'inline-block' : 'none'; }); +async function drawBody(people) { + // +} + +async function drawHand(hands) { + // +} + +async function runHumanDetect() { + const result = await human.detect(video, config); + drawFace(result.face); + drawBody(result.body); + drawHand(result.hand); + if (!paused) requestAnimationFrame(runHumanDetect); +} + +function setupGUI() { + const settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main')); + settings.addBoolean('Pause', paused, (val) => { paused = val; runHumanDetect(); }); + settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val); + settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val); + settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val); + settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val); + settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val); + settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val); + settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val); + settings.addRange('Max Objects', 1, 20, 5, 1, (val) => { + config.face.detector.maxFaces = parseInt(val); + config.body.maxDetections = parseInt(val); + }); + settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => { + config.face.detector.skipFrames = parseInt(val); + config.face.age.skipFrames = parseInt(val); + config.hand.skipFrames = parseInt(val); + }); + settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => { + config.face.detector.minConfidence = parseFloat(val); + config.hand.minConfidence = parseFloat(val); + }); + settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => { + config.face.detector.scoreThreshold = parseFloat(val); + config.hand.scoreThreshold = parseFloat(val); + config.body.scoreThreshold = parseFloat(val); + }); + settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => { + config.face.detector.iouThreshold = parseFloat(val); + config.hand.iouThreshold = parseFloat(val); + }); +} + +async function setupCanvas() { + canvas = document.getElementById('canvas'); + canvas.width = video.width; + canvas.height = video.height; + ctx = canvas.getContext('2d'); + ctx.fillStyle = 'lightblue'; + ctx.strokeStyle = 'lightblue'; + ctx.lineWidth = 1; + ctx.font = 'small-caps 1rem "Segoe UI"'; } async function setupCamera() { video = document.getElementById('video'); const stream = await navigator.mediaDevices.getUserMedia({ audio: false, - video: { facingMode: 'user', width: state.videoSize, height: state.videoSize }, + video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight }, }); video.srcObject = stream; return new Promise((resolve) => { - video.onloadedmetadata = () => resolve(video); + video.onloadedmetadata = () => { + resolve(video); + video.width = video.videoWidth; + video.height = video.videoHeight; + video.play(); + }; }); } async function main() { - await tf.setBackend(state.backend); - setupDatGui(); + await tf.setBackend('webgl'); + await tf.ready(); + await setupGUI(); await setupCamera(); - video.play(); - videoWidth = video.videoWidth; - videoHeight = video.videoHeight; - video.width = videoWidth; - video.height = videoHeight; - canvas = document.getElementById('output'); - canvas.width = videoWidth; - canvas.height = videoHeight; - const canvasContainer = document.querySelector('.canvas-wrapper'); - canvasContainer.style = `width: ${videoWidth}px; height: ${videoHeight}px`; - ctx = canvas.getContext('2d'); - // ctx.translate(canvas.width, 0); - // ctx.scale(-1, 1); - ctx.fillStyle = '#32EEDB'; - ctx.strokeStyle = '#32EEDB'; - ctx.lineWidth = 0.5; - human.load(options); - renderPrediction(); - if (state.renderPointcloud) { - document.querySelector('#scatter-gl-container').style = `width: ${state.videoSize}px; height: ${state.videoSize}px;`; - scatterGL = new ScatterGL(document.querySelector('#scatter-gl-container'), { rotateOnStart: false, selectEnabled: false }); - } + await setupCanvas(); + runHumanDetect(); } -main(); +window.onload = main; diff --git a/src/index.js b/src/index.js index 05b7e5ba..5b471c05 100644 --- a/src/index.js +++ b/src/index.js @@ -16,7 +16,7 @@ const models = { function mergeDeep(...objects) { const isObject = (obj) => obj && typeof obj === 'object'; return objects.reduce((prev, obj) => { - Object.keys(obj).forEach((key) => { + Object.keys(obj || {}).forEach((key) => { const pVal = prev[key]; const oVal = obj[key]; if (Array.isArray(pVal) && Array.isArray(oVal)) { @@ -79,3 +79,7 @@ async function detect(input, userConfig) { exports.detect = detect; exports.defaults = defaults; exports.models = models; +exports.facemesh = facemesh; +exports.ssrnet = ssrnet; +exports.posenet = posenet; +exports.handpose = handpose;