implemented blink detection

pull/50/head
Vladimir Mandic 2020-11-09 08:57:24 -05:00
parent 5a10835e90
commit 52d2bfa8da
24 changed files with 172 additions and 100 deletions

View File

@ -22,6 +22,7 @@
- [**Usage**](https://github.com/vladmandic/human/wiki/Usage)
- [**Configuration**](https://github.com/vladmandic/human/wiki/Configuration)
- [**Outputs**](https://github.com/vladmandic/human/wiki/Outputs)
- [**Gesture Recognition**](https://github.com/vladmandic/human/wiki/Gesture)
- [**Demos**](https://github.com/vladmandic/human/wiki/Demos)
- [**Development Server**](https://github.com/vladmandic/human/wiki/Development-Server)
- [**Build Process**](https://github.com/vladmandic/human/wiki/Build-Process)

View File

@ -297,7 +297,7 @@ async function detectSampleImages() {
}
function setupMenu() {
menu = new Menu(document.body, '...', { top: '1rem', right: '1rem' });
menu = new Menu(document.body, '', { top: '1rem', right: '1rem' });
const btn = menu.addButton('start video', 'pause video', () => detectVideo());
menu.addButton('process images', 'process images', () => detectSampleImages());
document.getElementById('play').addEventListener('click', () => btn.click());
@ -352,7 +352,7 @@ function setupMenu() {
menu.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu.addChart('FPS', 'FPS');
menuFX = new Menu(document.body, '...', { top: '1rem', right: '18rem' });
menuFX = new Menu(document.body, '', { top: '1rem', right: '18rem' });
menuFX.addLabel('ui options');
menuFX.addBool('crop & scale', ui, 'crop', () => setupCamera());
menuFX.addBool('camera front/back', ui, 'facing', () => setupCamera());

View File

@ -68,6 +68,31 @@ async function drawFace(result, canvas, ui, triangulation) {
ctx.fill(path);
}
}
// iris: array[center, left, top, right, bottom]
if (face.annotations && face.annotations.leftEyeIris) {
ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.leftEyeIris[4][1] - face.annotations.leftEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.leftEyeIris[0][0], face.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui.fillPolygons) {
ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
ctx.fill();
}
}
if (face.annotations && face.annotations.rightEyeIris) {
ctx.strokeStyle = ui.useDepth ? 'rgba(255, 200, 255, 0.3)' : ui.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.rightEyeIris[0][0], face.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui.fillPolygons) {
ctx.fillStyle = ui.useDepth ? 'rgba(255, 255, 200, 0.3)' : ui.baseColor;
ctx.fill();
}
}
}
}
}

View File

@ -55,6 +55,9 @@ function createCSS() {
input[type=range] { -webkit-appearance: none; }
input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; }
input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; }
.svg-background { fill:darkslategrey; cursor:pointer; opacity: 0.6; }
.svg-foreground { fill:white; cursor:pointer; opacity: 0.8; }
`;
const el = document.createElement('style');
el.innerHTML = css;
@ -89,18 +92,22 @@ class Menu {
this.container.id = `menu-container-${instance}`;
this.container.className = 'menu-container menu-container-fadein';
if (title !== '') {
const elTitle = document.createElement('div');
elTitle.className = 'menu-title';
elTitle.id = `menu-title-${instance}`;
elTitle.innerHTML = title;
this.menu.appendChild(elTitle);
elTitle.addEventListener('click', () => {
this.container.classList.toggle('menu-container-fadeout');
this.container.classList.toggle('menu-container-fadein');
this.menu.style.borderStyle = this.container.classList.contains('menu-container-fadeout') ? 'none' : 'solid';
});
}
// set menu title with pulldown arrow
const elTitle = document.createElement('div');
elTitle.className = 'menu-title';
elTitle.id = `menu-title-${instance}`;
const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" style="width: 2rem; height: 2rem; vertical-align: top;">
<path d="M400 32H48A48 48 0 0 0 0 80v352a48 48 0 0 0 48 48h352a48 48 0 0 0 48-48V80a48 48 0 0 0-48-48zm-51.37 182.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-background"/>
<path d="M348.63 214.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-foreground"/>
</svg>`;
elTitle.innerHTML = `${title}${svg}`;
this.menu.appendChild(elTitle);
elTitle.addEventListener('click', () => {
this.container.classList.toggle('menu-container-fadeout');
this.container.classList.toggle('menu-container-fadein');
this.menu.style.borderStyle = this.container.classList.contains('menu-container-fadeout') ? 'none' : 'solid';
});
this.menu.appendChild(this.container);
if (typeof parent === 'object') parent.appendChild(this.menu);
else document.getElementById(parent).appendChild(this.menu);

View File

@ -32958,8 +32958,13 @@ var rL = we((pc) => {
if (!n)
return [];
const t = [];
for (const e of n)
for (const e of n) {
e.annotations.rightCheek && e.annotations.leftCheek && e.annotations.rightCheek.length > 0 && e.annotations.leftCheek.length > 0 && t.push(`facing ${e.annotations.rightCheek[0][2] > 0 || e.annotations.leftCheek[0][2] < 0 ? "right" : "left"}`);
const r = Math.abs(e.mesh[374][1] - e.mesh[386][1]) / Math.abs(e.mesh[443][1] - e.mesh[450][1]);
r < 0.2 && t.push("blink left eye");
const i = Math.abs(e.mesh[145][1] - e.mesh[159][1]) / Math.abs(e.mesh[223][1] - e.mesh[230][1]);
i < 0.2 && t.push("blink right eye");
}
return t;
};
pc.hand = (n) => {
@ -33236,8 +33241,8 @@ class hG {
continue;
}
this.analyze("Start Age:"), this.config.async ? e = this.config.face.age.enabled ? xs.predict(o.image, this.config) : {} : (this.state = "run:age", t = ut(), e = this.config.face.age.enabled ? await xs.predict(o.image, this.config) : {}, this.perf.age = Math.trunc(ut() - t)), this.analyze("Start Gender:"), this.config.async ? r = this.config.face.gender.enabled ? Ls.predict(o.image, this.config) : {} : (this.state = "run:gender", t = ut(), r = this.config.face.gender.enabled ? await Ls.predict(o.image, this.config) : {}, this.perf.gender = Math.trunc(ut() - t)), this.analyze("Start Emotion:"), this.config.async ? i = this.config.face.emotion.enabled ? Ss.predict(o.image, this.config) : {} : (this.state = "run:emotion", t = ut(), i = this.config.face.emotion.enabled ? await Ss.predict(o.image, this.config) : {}, this.perf.emotion = Math.trunc(ut() - t)), this.analyze("End Emotion:"), this.config.async && ([e, r, i] = await Promise.all([e, r, i])), this.analyze("Finish Face:"), o.image.dispose();
const c = o.annotations.leftEyeIris && o.annotations.rightEyeIris ? Math.max(o.annotations.leftEyeIris[3][0] - o.annotations.leftEyeIris[1][0], o.annotations.rightEyeIris[3][0] - o.annotations.rightEyeIris[1][0]) : 0;
a.push({confidence: o.confidence, box: o.box, mesh: o.mesh, annotations: o.annotations, age: e.age, gender: r.gender, genderConfidence: r.confidence, emotion: i, iris: c !== 0 ? Math.trunc(100 * 11.7 / c) / 100 : 0}), this.analyze("End Face");
const c = o.annotations.leftEyeIris && o.annotations.rightEyeIris ? 11.7 * Math.max(Math.abs(o.annotations.leftEyeIris[3][0] - o.annotations.leftEyeIris[1][0]), Math.abs(o.annotations.rightEyeIris[4][1] - o.annotations.rightEyeIris[2][1])) : 0;
a.push({confidence: o.confidence, box: o.box, mesh: o.mesh, annotations: o.annotations, age: e.age, gender: r.gender, genderConfidence: r.confidence, emotion: i, iris: c !== 0 ? Math.trunc(c) / 100 : 0}), this.analyze("End Face");
}
return this.analyze("End FaceMesh:"), this.config.async && (this.perf.face && delete this.perf.face, this.perf.age && delete this.perf.age, this.perf.gender && delete this.perf.gender, this.perf.emotion && delete this.perf.emotion), a;
}
@ -33536,6 +33541,30 @@ async function drawFace(result, canvas, ui2, triangulation) {
ctx.fill(path);
}
}
if (face.annotations && face.annotations.leftEyeIris) {
ctx.strokeStyle = ui2.useDepth ? "rgba(255, 200, 255, 0.3)" : ui2.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.leftEyeIris[4][1] - face.annotations.leftEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.leftEyeIris[0][0], face.annotations.leftEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui2.fillPolygons) {
ctx.fillStyle = ui2.useDepth ? "rgba(255, 255, 200, 0.3)" : ui2.baseColor;
ctx.fill();
}
}
if (face.annotations && face.annotations.rightEyeIris) {
ctx.strokeStyle = ui2.useDepth ? "rgba(255, 200, 255, 0.3)" : ui2.baseColor;
ctx.beginPath();
const sizeX = Math.abs(face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0]) / 2;
const sizeY = Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]) / 2;
ctx.ellipse(face.annotations.rightEyeIris[0][0], face.annotations.rightEyeIris[0][1], sizeX, sizeY, 0, 0, 2 * Math.PI);
ctx.stroke();
if (ui2.fillPolygons) {
ctx.fillStyle = ui2.useDepth ? "rgba(255, 255, 200, 0.3)" : ui2.baseColor;
ctx.fill();
}
}
}
}
}
@ -33716,6 +33745,9 @@ function createCSS() {
input[type=range] { -webkit-appearance: none; }
input[type=range]::-webkit-slider-runnable-track { width: 100%; height: 1rem; cursor: pointer; background: ${theme.itemBackground}; border-radius: var(--rounded); border: 1px; }
input[type=range]::-webkit-slider-thumb { border: 1px solid #000000; margin-top: 0.05rem; height: 0.9rem; width: 1.5rem; border-radius: var(--rounded); background: ${theme.rangeBackground}; cursor: pointer; -webkit-appearance: none; }
.svg-background { fill:darkslategrey; cursor:pointer; opacity: 0.6; }
.svg-foreground { fill:white; cursor:pointer; opacity: 0.8; }
`;
const el = document.createElement("style");
el.innerHTML = css;
@ -33751,18 +33783,20 @@ class Menu {
this.container = document.createElement("div");
this.container.id = `menu-container-${instance}`;
this.container.className = "menu-container menu-container-fadein";
if (title !== "") {
const elTitle = document.createElement("div");
elTitle.className = "menu-title";
elTitle.id = `menu-title-${instance}`;
elTitle.innerHTML = title;
this.menu.appendChild(elTitle);
elTitle.addEventListener("click", () => {
this.container.classList.toggle("menu-container-fadeout");
this.container.classList.toggle("menu-container-fadein");
this.menu.style.borderStyle = this.container.classList.contains("menu-container-fadeout") ? "none" : "solid";
});
}
const elTitle = document.createElement("div");
elTitle.className = "menu-title";
elTitle.id = `menu-title-${instance}`;
const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" style="width: 2rem; height: 2rem; vertical-align: top;">
<path d="M400 32H48A48 48 0 0 0 0 80v352a48 48 0 0 0 48 48h352a48 48 0 0 0 48-48V80a48 48 0 0 0-48-48zm-51.37 182.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-background"/>
<path d="M348.63 214.31L232.06 348.16a10.38 10.38 0 0 1-16.12 0L99.37 214.31C92.17 206 97.28 192 107.43 192h233.14c10.15 0 15.26 14 8.06 22.31z" class="svg-foreground"/>
</svg>`;
elTitle.innerHTML = `${title}${svg}`;
this.menu.appendChild(elTitle);
elTitle.addEventListener("click", () => {
this.container.classList.toggle("menu-container-fadeout");
this.container.classList.toggle("menu-container-fadein");
this.menu.style.borderStyle = this.container.classList.contains("menu-container-fadeout") ? "none" : "solid";
});
this.menu.appendChild(this.container);
if (typeof parent === "object")
parent.appendChild(this.menu);
@ -34228,7 +34262,7 @@ async function detectSampleImages() {
status("");
}
function setupMenu() {
menu2 = new menu_default(document.body, "...", {top: "1rem", right: "1rem"});
menu2 = new menu_default(document.body, "", {top: "1rem", right: "1rem"});
const btn = menu2.addButton("start video", "pause video", () => detectVideo());
menu2.addButton("process images", "process images", () => detectSampleImages());
document.getElementById("play").addEventListener("click", () => btn.click());
@ -34279,7 +34313,7 @@ function setupMenu() {
});
menu2.addHTML('<hr style="min-width: 200px; border-style: inset; border-color: dimgray">');
menu2.addChart("FPS", "FPS");
menuFX = new menu_default(document.body, "...", {top: "1rem", right: "18rem"});
menuFX = new menu_default(document.body, "", {top: "1rem", right: "18rem"});
menuFX.addLabel("ui options");
menuFX.addBool("crop & scale", ui, "crop", () => setupCamera());
menuFX.addBool("camera front/back", ui, "facing", () => setupCamera());

File diff suppressed because one or more lines are too long

View File

@ -1,7 +1,7 @@
{
"inputs": {
"demo/browser.js": {
"bytes": 18569,
"bytes": 18563,
"imports": [
{
"path": "dist/human.esm.js"
@ -15,15 +15,15 @@
]
},
"demo/draw.js": {
"bytes": 7627,
"bytes": 9119,
"imports": []
},
"demo/menu.js": {
"bytes": 12725,
"bytes": 13466,
"imports": []
},
"dist/human.esm.js": {
"bytes": 1278262,
"bytes": 1278518,
"imports": []
}
},
@ -31,28 +31,28 @@
"dist/demo-browser-index.js.map": {
"imports": [],
"inputs": {},
"bytes": 5530763
"bytes": 5534714
},
"dist/demo-browser-index.js": {
"imports": [],
"inputs": {
"dist/human.esm.js": {
"bytesInOutput": 1664653
"bytesInOutput": 1664968
},
"dist/human.esm.js": {
"bytesInOutput": 8716
},
"demo/draw.js": {
"bytesInOutput": 7451
"bytesInOutput": 8896
},
"demo/menu.js": {
"bytesInOutput": 12727
"bytesInOutput": 13425
},
"demo/browser.js": {
"bytesInOutput": 16215
"bytesInOutput": 16209
}
},
"bytes": 1709884
"bytes": 1712336
}
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -194,7 +194,7 @@
]
},
"src/gesture.js": {
"bytes": 2175,
"bytes": 2680,
"imports": []
},
"src/hand/anchors.js": {
@ -243,7 +243,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13725,
"bytes": 13770,
"imports": [
{
"path": "src/face/facemesh.js"
@ -301,7 +301,7 @@
"dist/human.esm-nobundle.js.map": {
"imports": [],
"inputs": {},
"bytes": 620133
"bytes": 620981
},
"dist/human.esm-nobundle.js": {
"imports": [],
@ -394,7 +394,7 @@
"bytesInOutput": 1219
},
"src/gesture.js": {
"bytesInOutput": 1219
"bytesInOutput": 1459
},
"src/imagefx.js": {
"bytesInOutput": 11013
@ -409,13 +409,13 @@
"bytesInOutput": 3047
},
"src/human.js": {
"bytesInOutput": 7240
"bytesInOutput": 7256
},
"src/human.js": {
"bytesInOutput": 0
}
},
"bytes": 216588
"bytes": 216844
}
}
}

4
dist/human.esm.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

12
dist/human.esm.json vendored
View File

@ -371,7 +371,7 @@
]
},
"src/gesture.js": {
"bytes": 2175,
"bytes": 2680,
"imports": []
},
"src/hand/anchors.js": {
@ -433,7 +433,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13725,
"bytes": 13770,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,7 +513,7 @@
"dist/human.esm.js.map": {
"imports": [],
"inputs": {},
"bytes": 5416118
"bytes": 5416966
},
"dist/human.esm.js": {
"imports": [],
@ -663,7 +663,7 @@
"bytesInOutput": 1197
},
"src/gesture.js": {
"bytesInOutput": 1220
"bytesInOutput": 1460
},
"src/imagefx.js": {
"bytesInOutput": 11014
@ -678,13 +678,13 @@
"bytesInOutput": 3048
},
"src/human.js": {
"bytesInOutput": 7258
"bytesInOutput": 7274
},
"src/human.js": {
"bytesInOutput": 0
}
},
"bytes": 1278262
"bytes": 1278518
}
}
}

4
dist/human.js vendored

File diff suppressed because one or more lines are too long

4
dist/human.js.map vendored

File diff suppressed because one or more lines are too long

12
dist/human.json vendored
View File

@ -371,7 +371,7 @@
]
},
"src/gesture.js": {
"bytes": 2175,
"bytes": 2680,
"imports": []
},
"src/hand/anchors.js": {
@ -433,7 +433,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13725,
"bytes": 13770,
"imports": [
{
"path": "node_modules/@tensorflow/tfjs/dist/tf.node.js"
@ -513,7 +513,7 @@
"dist/human.js.map": {
"imports": [],
"inputs": {},
"bytes": 5416114
"bytes": 5416962
},
"dist/human.js": {
"imports": [],
@ -663,7 +663,7 @@
"bytesInOutput": 1197
},
"src/gesture.js": {
"bytesInOutput": 1220
"bytesInOutput": 1460
},
"src/imagefx.js": {
"bytesInOutput": 11014
@ -678,10 +678,10 @@
"bytesInOutput": 3047
},
"src/human.js": {
"bytesInOutput": 7296
"bytesInOutput": 7312
}
},
"bytes": 1278307
"bytes": 1278563
}
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

4
dist/human.node.js vendored

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

12
dist/human.node.json vendored
View File

@ -194,7 +194,7 @@
]
},
"src/gesture.js": {
"bytes": 2175,
"bytes": 2680,
"imports": []
},
"src/hand/anchors.js": {
@ -243,7 +243,7 @@
"imports": []
},
"src/human.js": {
"bytes": 13725,
"bytes": 13770,
"imports": [
{
"path": "src/face/facemesh.js"
@ -301,7 +301,7 @@
"dist/human.node-nobundle.js.map": {
"imports": [],
"inputs": {},
"bytes": 634264
"bytes": 635158
},
"dist/human.node-nobundle.js": {
"imports": [],
@ -394,7 +394,7 @@
"bytesInOutput": 1219
},
"src/gesture.js": {
"bytesInOutput": 1219
"bytesInOutput": 1459
},
"src/imagefx.js": {
"bytesInOutput": 11013
@ -412,10 +412,10 @@
"bytesInOutput": 28
},
"src/human.js": {
"bytesInOutput": 7240
"bytesInOutput": 7256
}
},
"bytes": 216595
"bytes": 216851
}
}
}

View File

@ -25,6 +25,10 @@ exports.face = (res) => {
if (face.annotations['rightCheek'] && face.annotations['leftCheek'] && (face.annotations['rightCheek'].length > 0) && (face.annotations['leftCheek'].length > 0)) {
gestures.push(`facing ${((face.annotations['rightCheek'][0][2] > 0) || (face.annotations['leftCheek'][0][2] < 0)) ? 'right' : 'left'}`);
}
const openLeft = Math.abs(face.mesh[374][1] - face.mesh[386][1]) / Math.abs(face.mesh[443][1] - face.mesh[450][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openLeft < 0.2) gestures.push('blink left eye');
const openRight = Math.abs(face.mesh[145][1] - face.mesh[159][1]) / Math.abs(face.mesh[223][1] - face.mesh[230][1]); // center of eye inner lid y coord div center of wider eye border y coord
if (openRight < 0.2) gestures.push('blink right eye');
}
return gestures;
};

View File

@ -245,9 +245,10 @@ class Human {
face.image.dispose();
// calculate iris distance
// iris: array[ bottom, left, top, right, center ]
const iris = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
? Math.max(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0], face.annotations.rightEyeIris[3][0] - face.annotations.rightEyeIris[1][0])
// iris: array[ center, left, top, right, bottom]
const irisSize = (face.annotations.leftEyeIris && face.annotations.rightEyeIris)
/* average human iris size is 11.7mm */
? 11.7 * Math.max(Math.abs(face.annotations.leftEyeIris[3][0] - face.annotations.leftEyeIris[1][0]), Math.abs(face.annotations.rightEyeIris[4][1] - face.annotations.rightEyeIris[2][1]))
: 0;
// combine results
@ -260,7 +261,7 @@ class Human {
gender: genderRes.gender,
genderConfidence: genderRes.confidence,
emotion: emotionRes,
iris: (iris !== 0) ? Math.trunc(100 * 11.7 /* human iris size in mm */ / iris) / 100 : 0,
iris: (irisSize !== 0) ? Math.trunc(irisSize) / 100 : 0,
});
this.analyze('End Face');
}

2
wiki

@ -1 +1 @@
Subproject commit b70041ff6b67e01ac2ed521bdd01c43a970ff351
Subproject commit 63c2347b5256777d5154bd5d502b2a075d3d950d