fix and optimize for mobile platform

CHANGELOG.md

@@ -9,11 +9,12 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ## Changelog
 
+### **HEAD -> main** 2021/05/27 mandic00@live.com
+
+
 ### **1.9.4** 2021/05/27 mandic00@live.com
 
-
+- fix demo facecompare
-### **origin/main** 2021/05/26 mandic00@live.com
-
 - webhint and lighthouse optimizations
 - add camera startup diag messages
 - implemented unified result.persons that combines face, body and hands for each person

demo/index.html

@@ -75,10 +75,10 @@
     <div id="loader" class="loader"></div>
     <div id="status" class="status"></div>
     <div id="menubar" class="menubar">
-      <div id="btnDisplay" class="icon"><div class="icon-binoculars"> </div>display options</div>
+      <div id="btnDisplay" class="icon"><div class="icon-binoculars"> </div>display</div>
-      <div id="btnImage" class="icon"><div class="icon-brush"></div>image processing</div>
+      <div id="btnImage" class="icon"><div class="icon-brush"></div>input</div>
-      <div id="btnProcess" class="icon"><div class="icon-stats"></div>model processing</div>
+      <div id="btnProcess" class="icon"><div class="icon-stats"></div>options</div>
-      <div id="btnModel" class="icon"><div class="icon-games"></div>model selection</div>
+      <div id="btnModel" class="icon"><div class="icon-games"></div>models</div>
       <div id="btnStart" class="icon"><div class="icon-webcam"></div><span id="btnStartText">start video</span></div>
     </div>
     <div id="media">

demo/index.js

@@ -16,7 +16,6 @@ let human;
 
 const userConfig = {
   warmup: 'none',
-  /*
   backend: 'webgl',
   async: false,
   cacheSensitivity: 0,
@@ -31,12 +30,11 @@ const userConfig = {
     description: { enabled: false },
     emotion: { enabled: false },
   },
-  hand: { enabled: true },
+  hand: { enabled: false },
-  body: { enabled: true, modelPath: 'posenet.json' },
+  body: { enabled: false, modelPath: 'posenet.json' },
   // body: { enabled: true, modelPath: 'blazepose.json' },
   object: { enabled: false },
   gesture: { enabled: true },
-  */
 };
 
 const drawOptions = {
@@ -48,7 +46,7 @@ const drawOptions = {
 const ui = {
   // configurable items
   console: true, // log messages to browser console
-  crop: true, // video mode crop to size or leave full frame
+  crop: false, // video mode crop to size or leave full frame
   facing: true, // camera facing front or back
   baseBackground: 'rgba(50, 50, 50, 1)', // 'grey'
   columns: 2, // when processing sample images create this many columns
@@ -305,23 +303,23 @@ async function setupCamera() {
     ui.busy = false;
     return msg;
   }
-  // enumerate devices for diag purposes
-  if (initialCameraAccess) {
-    const devices = await navigator.mediaDevices.enumerateDevices();
-    log('enumerated devices:');
-    for (const device of devices) log(`  kind:${device.kind} label:${device.label} id:${device.deviceId}`);
-  }
   let stream;
   const constraints = {
     audio: false,
     video: {
       facingMode: ui.facing ? 'user' : 'environment',
       resizeMode: ui.crop ? 'crop-and-scale' : 'none',
+      width: { ideal: document.body.clientWidth },
+      // height: { ideal: document.body.clientHeight }, // not set as we're using aspectRation to get height instead
+      aspectRatio: document.body.clientWidth / document.body.clientHeight,
       // deviceId: 'xxxx' // if you have multiple webcams, specify one to use explicitly
     },
   };
-  if (window.innerWidth > window.innerHeight) constraints.video.width = { ideal: window.innerWidth };
+  // enumerate devices for diag purposes
-  else constraints.video.height = { ideal: (window.innerHeight - document.getElementById('menubar').offsetHeight) };
+  if (initialCameraAccess) {
+    navigator.mediaDevices.enumerateDevices().then((devices) => log('enumerated devices:', devices));
+    log('camera constraints', constraints);
+  }
   try {
     stream = await navigator.mediaDevices.getUserMedia(constraints);
   } catch (err) {
@@ -334,37 +332,24 @@ async function setupCamera() {
     ui.busy = false;
     return msg;
   }
-  if (stream) video.srcObject = stream;
-  else {
-    ui.busy = false;
-    return 'camera stream empty';
-  }
   const tracks = stream.getVideoTracks();
   if (tracks && tracks.length >= 1) {
-    if (tracks.length > 1) {
+    if (initialCameraAccess) log('enumerated viable tracks:', tracks);
-      log('enumerated viable tracks:', tracks.length);
-      for (const t of tracks) log(`  ${t.kind}: ${t.label}`);
-    }
   } else {
     ui.busy = false;
     return 'no camera track';
   }
   const track = stream.getVideoTracks()[0];
   const settings = track.getSettings();
-  if (initialCameraAccess) log('selected camera:', track.label, 'id:', settings.deviceId);
+  if (initialCameraAccess) log('selected video source:', track, settings); // log('selected camera:', track.label, 'id:', settings.deviceId);
-  // log('camera constraints:', constraints, 'window:', { width: window.innerWidth, height: window.innerHeight }, 'settings:', settings, 'track:', track);
+  ui.camera = { name: track.label.toLowerCase(), width: video.videoWidth, height: video.videoHeight, facing: settings.facingMode === 'user' ? 'front' : 'back' };
-  ui.camera = { name: track.label.toLowerCase(), width: settings.width, height: settings.height, facing: settings.facingMode === 'user' ? 'front' : 'back' };
   initialCameraAccess = false;
-  return new Promise((resolve) => {
+  const promise = !stream || new Promise((resolve) => {
-    video.onloadeddata = async () => {
+    video.onloadeddata = () => {
-      video.width = video.videoWidth;
+      if (settings.width > settings.height) canvas.style.width = '100vw';
-      video.height = video.videoHeight;
+      else canvas.style.height = '100vh';
-      canvas.width = video.width;
+      ui.menuWidth.input.setAttribute('value', video.videoWidth);
-      canvas.height = video.height;
+      ui.menuHeight.input.setAttribute('value', video.videoHeight);
-      canvas.style.width = canvas.width > canvas.height ? '100vw' : '';
-      canvas.style.height = canvas.width > canvas.height ? '' : '100vh';
-      ui.menuWidth.input.setAttribute('value', video.width);
-      ui.menuHeight.input.setAttribute('value', video.height);
       if (live) video.play();
       // eslint-disable-next-line no-use-before-define
       if (live && !ui.detectThread) runHumanDetect(video, canvas);
@@ -372,6 +357,13 @@ async function setupCamera() {
       resolve();
     };
   });
+  // attach input to video element
+  if (stream) {
+    video.srcObject = stream;
+    return promise;
+  }
+  ui.busy = false;
+  return 'camera stream empty';
 }
 
 function initPerfMonitor() {
@@ -645,7 +637,9 @@ function setupMenu() {
 
 async function resize() {
   window.onresize = null;
-  const viewportScale = 1; // Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
+  // best setting for mobile, ignored for desktop
+  // can set dynamic value such as Math.min(1, Math.round(100 * window.innerWidth / 960) / 100);
+  const viewportScale = 0.7;
   if (!ui.viewportSet) {
     const viewport = document.querySelector('meta[name=viewport]');
     viewport.setAttribute('content', `width=device-width, shrink-to-fit=yes, minimum-scale=0.2, maximum-scale=2.0, user-scalable=yes, initial-scale=${viewportScale}`);
@@ -664,10 +658,10 @@ async function resize() {
   menu.process.menu.style.left = x[2];
   menu.models.menu.style.left = x[3];
 
-  const fontSize = Math.trunc(10 * (1 - viewportScale)) + 16;
+  const fontSize = Math.trunc(10 * (1 - viewportScale)) + 14;
   document.documentElement.style.fontSize = `${fontSize}px`;
-
+  human.draw.options.font = `small-caps ${fontSize}px "Segoe UI"`;
-  human.draw.options.font = `small-caps ${fontSize + 4}px "Segoe UI"`;
+  human.draw.options.lineHeight = fontSize + 2;
 
   await setupCamera();
   window.onresize = resize;
@@ -686,7 +680,7 @@ async function main() {
   window.addEventListener('unhandledrejection', (evt) => {
     // eslint-disable-next-line no-console
     console.error(evt.reason || evt);
-    document.getElementById('log').innerHTML = evt?.reason?.message || evt?.reason || evt;
+    document.getElementById('log').innerHTML = evt.reason.message || evt.reason || evt;
     status('exception error');
     evt.preventDefault();
   });

src/draw/draw.ts

@@ -51,7 +51,7 @@ export const options: DrawOptions = {
   color: <string>'rgba(173, 216, 230, 0.3)', // 'lightblue' with light alpha channel
   labelColor: <string>'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel
   shadowColor: <string>'black',
-  font: <string>'small-caps 16px "Segoe UI"',
+  font: <string>'small-caps 14px "Segoe UI"',
   lineHeight: <number>24,
   lineWidth: <number>6,
   pointSize: <number>2,

src/gesture/gesture.ts

@@ -65,20 +65,22 @@ export const iris = (res): Gesture[] => {
       gestures.push({ iris: i, gesture: 'facing center' });
     }
 
-    const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].annotations.rightEyeIris[0][0];
+    const rightIrisCenterX = Math.abs(res[i].mesh[33][0] - res[i].annotations.rightEyeIris[0][0]) / res[i].box[2];
-    const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].annotations.leftEyeIris[0][0];
+    const leftIrisCenterX = Math.abs(res[i].mesh[263][0] - res[i].annotations.leftEyeIris[0][0]) / res[i].box[2];
-    if (leftIrisCenterX > 0.033 || rightIrisCenterX > 0.033) center = false;
+    if (leftIrisCenterX > 0.06 || rightIrisCenterX > 0.06) center = false;
-    if (leftIrisCenterX > 0.033) gestures.push({ iris: i, gesture: 'looking right' });
+    if (leftIrisCenterX > 0.06) gestures.push({ iris: i, gesture: 'looking right' });
-    if (rightIrisCenterX > 0.033) gestures.push({ iris: i, gesture: 'looking left' });
+    if (rightIrisCenterX > 0.06) gestures.push({ iris: i, gesture: 'looking left' });
 
-    const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].annotations.rightEyeIris[0][1];
+    const rightIrisCenterY = Math.abs(res[i].mesh[145][1] - res[i].annotations.rightEyeIris[0][1]) / res[i].box[3];
-    const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].annotations.leftEyeIris[0][1];
+    const leftIrisCenterY = Math.abs(res[i].mesh[374][1] - res[i].annotations.leftEyeIris[0][1]) / res[i].box[3];
-    if (leftIrisCenterY < 0.015 || rightIrisCenterY < 0.015 || leftIrisCenterY > 0.030 || rightIrisCenterY > 0.030) center = false;
+    if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01 || leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) center = false;
-    if (leftIrisCenterY < 0.015 || rightIrisCenterY < 0.015) gestures.push({ iris: i, gesture: 'looking down' });
+    if (leftIrisCenterY < 0.01 || rightIrisCenterY < 0.01) gestures.push({ iris: i, gesture: 'looking down' });
-    if (leftIrisCenterY > 0.030 || rightIrisCenterY > 0.030) gestures.push({ iris: i, gesture: 'looking up' });
+    if (leftIrisCenterY > 0.025 || rightIrisCenterY > 0.025) gestures.push({ iris: i, gesture: 'looking up' });
 
     // still center;
     if (center) gestures.push({ iris: i, gesture: 'looking center' });
+
+    console.log(leftIrisCenterX, rightIrisCenterX, leftIrisCenterY, rightIrisCenterY, gestures);
   }
   return gestures;
 };