ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

updated demo

pull/50/head
Vladimir Mandic 2020-10-11 21:21:41 -04:00
parent de01483bc5
commit 937a97f0d6
9 changed files with 3014 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
demo/index.html
View File

@ -1,25 +1,12 @@
<head> <head>
<script src="https://cdn.jsdelivr.net/npm/three@0.106.2/build/three.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/scatter-gl@0.0.1/lib/scatter-gl.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tensorflow/2.6.0/tf.es2017.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/tensorflow/2.6.0/tf.es2017.min.js"></script>
<style> <script src="https://cdn.jsdelivr.net/npm/quicksettings@latest/quicksettings.min.js"></script>
.canvas-wrapper { display: inline-block; vertical-align: top; } <script src="../dist/human.js"></script>
#scatter-gl-container { display: inline-block; vertical-align: top; border: solid 1px black; position: relative; } <script src="./index.js"></script>
#scatter-gl-container canvas { transform: translate3d(-50%, -50%, 0); left: 50%; top: 50%; position: absolute; }
</style>
</head> </head>
<body> <body style="margin: 0; background: black">
<div id="main"> <div id="main">
<div class="container"> <video id="video" playsinline style="display: none"></video>
<div class="canvas-wrapper"> <canvas id="canvas"></canvas>
<canvas id="output"></canvas>
<video id="video" playsinline style="visibility: hidden; width: auto; height: auto">
</video>
</div>
<div id="scatter-gl-container"></div>
<div id="faces"></div>
</div>
</div> </div>
</body> </body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.6/dat.gui.min.js"></script>
<script type="module" src="./index.js"></script>

189
demo/index.js
View File

@ -1,120 +1,127 @@
/* global tf, ScatterGL, dat */ /* eslint-disable no-return-assign */
/* global tf, human, QuickSettings */
import human from '../dist/human.esm.js'; let paused = false;
const state = {
backend: 'webgl',
triangulateMesh: true,
renderPointcloud: true,
stop: false,
videoSize: 700,
};
const options = {
};
let ctx;
let videoWidth;
let videoHeight;
let video; let video;
let canvas; let canvas;
let scatterGLHasInitialized = false; let ctx;
let scatterGL;
async function renderPrediction() { const config = {
const predictions = await human.detect(video); face: {
ctx.drawImage(video, 0, 0, videoWidth, videoHeight, 0, 0, canvas.width, canvas.height); enabled: true,
const div = document.getElementById('faces'); detector: { maxFaces: 10, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
div.innerHTML = ''; mesh: { enabled: true },
for (const prediction of predictions) { iris: { enabled: true },
div.appendChild(prediction.canvas); age: { enabled: false, skipFrames: 5 },
gender: { enabled: false },
},
body: { enabled: false, maxDetections: 5, scoreThreshold: 0.75, nmsRadius: 20 },
hand: { enabled: false, skipFrames: 5, minConfidence: 0.8, iouThreshold: 0.3, scoreThreshold: 0.75 },
};
async function drawFace(faces) {
for (const face of faces) {
ctx.drawImage(video, 0, 0, video.width, video.height, 0, 0, canvas.width, canvas.height);
ctx.beginPath(); ctx.beginPath();
ctx.rect(prediction.box[0], prediction.box[1], prediction.box[2], prediction.box[3]); ctx.rect(face.box[0], face.box[1], face.box[2], face.box[3]);
ctx.font = 'small-caps 1rem "Segoe UI"'; ctx.fillText(`face ${face.gender || ''} ${face.age || ''} ${face.iris ? 'iris: ' + face.iris : ''}`, face.box[0] + 2, face.box[1] + 16, face.box[2]);
ctx.fillText(`${prediction.gender} ${prediction.age}`, prediction.box[0] + 2, prediction.box[1] + 16, prediction.box[2]);
ctx.stroke(); ctx.stroke();
if (state.triangulateMesh) { if (face.mesh) {
for (let i = 0; i < human.triangulation.length / 3; i++) { for (const point of face.mesh) {
const points = [human.triangulation[i * 3], human.triangulation[i * 3 + 1], human.triangulation[i * 3 + 2]].map((index) => prediction.mesh[index]); ctx.fillStyle = `rgba(${127.5 + (2 * point[2])}, ${127.5 - (2 * point[2])}, 255, 0.5)`;
const region = new Path2D();
region.moveTo(points[0][0], points[0][1]);
for (let j = 1; i < points.length; j++) region.lineTo(points[j][0], points[j][1]);
region.closePath();
ctx.stroke(region);
}
} else {
for (let i = 0; i < prediction.mesh.length; i++) {
const x = prediction.mesh[i][0];
const y = prediction.mesh[i][1];
ctx.beginPath(); ctx.beginPath();
ctx.arc(x, y, 1 /* radius */, 0, 2 * Math.PI); ctx.arc(point[0], point[1], 1 /* radius */, 0, 2 * Math.PI);
ctx.fill(); ctx.fill();
} }
} }
if (state.renderPointcloud && scatterGL != null) {
const pointsData = predictions.map((pred) => pred.mesh.map((point) => ([-point[0], -point[1], -point[2]])));
let flattenedPointsData = [];
for (let i = 0; i < pointsData.length; i++) {
flattenedPointsData = flattenedPointsData.concat(pointsData[i]);
}
const dataset = new ScatterGL.Dataset(flattenedPointsData);
if (!scatterGLHasInitialized) scatterGL.render(dataset);
else scatterGL.updateDataset(dataset);
scatterGLHasInitialized = true;
}
} }
if (!state.stop) requestAnimationFrame(renderPrediction);
} }
function setupDatGui() { async function drawBody(people) {
const gui = new dat.GUI(); //
gui.add(state, 'stop').onChange(() => { renderPrediction(); }); }
gui.add(state, 'backend', ['webgl', 'cpu']).onChange((backend) => { tf.setBackend(backend); });
gui.add(options, 'maxFaces', 1, 100, 1).onChange(() => { human.load(options); }); async function drawHand(hands) {
gui.add(options, 'detectionConfidence', 0, 1, 0.05).onChange(() => { human.load(options); }); //
gui.add(options, 'iouThreshold', 0, 1, 0.05).onChange(() => { human.load(options); }); }
gui.add(options, 'scoreThreshold', 0, 1, 0.05).onChange(() => { human.load(options); });
gui.add(state, 'triangulateMesh'); async function runHumanDetect() {
gui.add(state, 'renderPointcloud').onChange((render) => { document.querySelector('#scatter-gl-container').style.display = render ? 'inline-block' : 'none'; }); const result = await human.detect(video, config);
drawFace(result.face);
drawBody(result.body);
drawHand(result.hand);
if (!paused) requestAnimationFrame(runHumanDetect);
}
function setupGUI() {
const settings = QuickSettings.create(10, 10, 'Settings', document.getElementById('main'));
settings.addBoolean('Pause', paused, (val) => { paused = val; runHumanDetect(); });
settings.addBoolean('Face Detect', config.face.enabled, (val) => config.face.enabled = val);
settings.addBoolean('Face Mesh', config.face.mesh.enabled, (val) => config.face.mesh.enabled = val);
settings.addBoolean('Face Iris', config.face.iris.enabled, (val) => config.face.iris.enabled = val);
settings.addBoolean('Face Age', config.face.age.enabled, (val) => config.face.age.enabled = val);
settings.addBoolean('Face Gender', config.face.gender.enabled, (val) => config.face.gender.enabled = val);
settings.addBoolean('Body Pose', config.body.enabled, (val) => config.body.enabled = val);
settings.addBoolean('Hand Pose', config.hand.enabled, (val) => config.hand.enabled = val);
settings.addRange('Max Objects', 1, 20, 5, 1, (val) => {
config.face.detector.maxFaces = parseInt(val);
config.body.maxDetections = parseInt(val);
});
settings.addRange('Skip Frames', 1, 20, config.face.detector.skipFrames, 1, (val) => {
config.face.detector.skipFrames = parseInt(val);
config.face.age.skipFrames = parseInt(val);
config.hand.skipFrames = parseInt(val);
});
settings.addRange('Min Confidence', 0.1, 1.0, config.face.detector.minConfidence, 0.05, (val) => {
config.face.detector.minConfidence = parseFloat(val);
config.hand.minConfidence = parseFloat(val);
});
settings.addRange('Score Threshold', 0.1, 1.0, config.face.detector.scoreThreshold, 0.05, (val) => {
config.face.detector.scoreThreshold = parseFloat(val);
config.hand.scoreThreshold = parseFloat(val);
config.body.scoreThreshold = parseFloat(val);
});
settings.addRange('IOU Threshold', 0.1, 1.0, config.face.detector.iouThreshold, 0.05, (val) => {
config.face.detector.iouThreshold = parseFloat(val);
config.hand.iouThreshold = parseFloat(val);
});
}
async function setupCanvas() {
canvas = document.getElementById('canvas');
canvas.width = video.width;
canvas.height = video.height;
ctx = canvas.getContext('2d');
ctx.fillStyle = 'lightblue';
ctx.strokeStyle = 'lightblue';
ctx.lineWidth = 1;
ctx.font = 'small-caps 1rem "Segoe UI"';
} }
async function setupCamera() { async function setupCamera() {
video = document.getElementById('video'); video = document.getElementById('video');
const stream = await navigator.mediaDevices.getUserMedia({ const stream = await navigator.mediaDevices.getUserMedia({
audio: false, audio: false,
video: { facingMode: 'user', width: state.videoSize, height: state.videoSize }, video: { facingMode: 'user', width: window.innerWidth, height: window.innerHeight },
}); });
video.srcObject = stream; video.srcObject = stream;
return new Promise((resolve) => { return new Promise((resolve) => {
video.onloadedmetadata = () => resolve(video); video.onloadedmetadata = () => {
resolve(video);
video.width = video.videoWidth;
video.height = video.videoHeight;
video.play();
};
}); });
} }
async function main() { async function main() {
await tf.setBackend(state.backend); await tf.setBackend('webgl');
setupDatGui(); await tf.ready();
await setupGUI();
await setupCamera(); await setupCamera();
video.play(); await setupCanvas();
videoWidth = video.videoWidth; runHumanDetect();
videoHeight = video.videoHeight;
video.width = videoWidth;
video.height = videoHeight;
canvas = document.getElementById('output');
canvas.width = videoWidth;
canvas.height = videoHeight;
const canvasContainer = document.querySelector('.canvas-wrapper');
canvasContainer.style = `width: ${videoWidth}px; height: ${videoHeight}px`;
ctx = canvas.getContext('2d');
// ctx.translate(canvas.width, 0);
// ctx.scale(-1, 1);
ctx.fillStyle = '#32EEDB';
ctx.strokeStyle = '#32EEDB';
ctx.lineWidth = 0.5;
human.load(options);
renderPrediction();
if (state.renderPointcloud) {
document.querySelector('#scatter-gl-container').style = `width: ${state.videoSize}px; height: ${state.videoSize}px;`;
scatterGL = new ScatterGL(document.querySelector('#scatter-gl-container'), { rotateOnStart: false, selectEnabled: false });
}
} }
main(); window.onload = main;

2661
dist/human.esm.js vendored
View File

File diff suppressed because it is too large Load Diff

6
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

478
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
src/facemesh/index.js
View File

@ -3,8 +3,10 @@ const blazeface = require('../blazeface');
const keypoints = require('./keypoints'); const keypoints = require('./keypoints');
const pipe = require('./pipeline'); const pipe = require('./pipeline');
const uv_coords = require('./uvcoords'); const uv_coords = require('./uvcoords');
const triangulation = require('./triangulation').default;
exports.uv_coords = uv_coords; exports.uv_coords = uv_coords;
exports.triangulation = triangulation;
async function loadDetectorModel(config) { async function loadDetectorModel(config) {
return blazeface.load(config); return blazeface.load(config);

0
src/triangulation.js → src/facemesh/triangulation.js
View File

6
src/index.js
View File

@ -16,7 +16,7 @@ const models = {
function mergeDeep(...objects) { function mergeDeep(...objects) {
const isObject = (obj) => obj && typeof obj === 'object'; const isObject = (obj) => obj && typeof obj === 'object';
return objects.reduce((prev, obj) => { return objects.reduce((prev, obj) => {
Object.keys(obj).forEach((key) => { Object.keys(obj || {}).forEach((key) => {
const pVal = prev[key]; const pVal = prev[key];
const oVal = obj[key]; const oVal = obj[key];
if (Array.isArray(pVal) && Array.isArray(oVal)) { if (Array.isArray(pVal) && Array.isArray(oVal)) {
@ -79,3 +79,7 @@ async function detect(input, userConfig) {
exports.detect = detect; exports.detect = detect;
exports.defaults = defaults; exports.defaults = defaults;
exports.models = models; exports.models = models;
exports.facemesh = facemesh;
exports.ssrnet = ssrnet;
exports.posenet = posenet;
exports.handpose = handpose;