added face3d demo

CHANGELOG.md

@@ -11,6 +11,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
 
 ### **HEAD -> main** 2021/03/29 mandic00@live.com
 
+- enable buffering
 - new icons
 - new serve module and demo structure
 

README.md

@@ -16,8 +16,8 @@
 JavaScript module using TensorFlow/JS Machine Learning library  
 
 - **Browser**:  
-  Compatible with *CPU*, *WebGL*, *WASM* backends  
   Compatible with both desktop and mobile platforms  
+  Compatible with *CPU*, *WebGL*, *WASM* backends  
   Compatible with *WebWorker* execution
 - **NodeJS**:  
   Compatible with both software *tfjs-node* and  
@@ -27,9 +27,14 @@ Check out [**Live Demo**](https://vladmandic.github.io/human/demo/index.html) fo
 
 <br>
 
+## Demos
+
+- [**Demo Application**](https://vladmandic.github.io/human/demo/index.html)
+- [**Face Extraction, Description, Identification and Matching**](https://vladmandic.github.io/human/demo/facematch.html)
+- [**Face Extraction and 3D Rendering**](https://vladmandic.github.io/human/demo/face3d.html)
+
 ## Project pages
 
-- [**Live Demo**](https://vladmandic.github.io/human/demo/index.html)
 - [**Code Repository**](https://github.com/vladmandic/human)
 - [**NPM Package**](https://www.npmjs.com/package/@vladmandic/human)
 - [**Issues Tracker**](https://github.com/vladmandic/human/issues)
@@ -98,6 +103,10 @@ As presented in the demo application...
 
 ![Face Matching](assets/screenshot-facematch.jpg)
 
+**Face3D OpenGL Rendering:**
+
+![Face Matching](assets/screenshot-face3d.jpg)
+
 **468-Point Face Mesh Defails:**  
 
 ![FaceMesh](assets/facemesh.png)

demo/face3d.js

@@ -1,6 +1,4 @@
-// @ts-nocheck
+import { DoubleSide, Mesh, MeshBasicMaterial, OrthographicCamera, Scene, sRGBEncoding, VideoTexture, WebGLRenderer, BufferGeometry, BufferAttribute } from './helpers/three.js';
-
-import { DoubleSide, Mesh, MeshBasicMaterial, OrthographicCamera, Scene, sRGBEncoding, VideoTexture, WebGLRenderer, BufferGeometry, BufferAttribute, Vector3, Triangle, Matrix4 } from './helpers/three.js';
 import { OrbitControls } from './helpers/three-orbitControls.js';
 import Human from '../dist/human.esm.js'; // equivalent of @vladmandic/human
 
@@ -12,6 +10,7 @@ const userConfig = {
   videoOptimized: false,
   filter: { enabled: false },
   face: { enabled: true,
+    detector: { rotation: false, maxFaces: 1 },
     mesh: { enabled: true },
     iris: { enabled: true },
     description: { enabled: false },
@@ -24,82 +23,6 @@ const userConfig = {
 };
 const human = new Human(userConfig);
 
-class FaceGeometry extends BufferGeometry {
-  constructor(inIndices, inUVMap) {
-    super();
-    this.indices = inIndices;
-    this.texCoords = inUVMap;
-    this.positions = new Float32Array(468 * 3);
-    this.uvs = new Float32Array(468 * 2);
-    this.setAttribute('position', new BufferAttribute(this.positions, 3));
-    this.setAttribute('uv', new BufferAttribute(this.uvs, 2));
-    this.setUvs();
-    this.setIndex(this.indices);
-    this.computeVertexNormals();
-    this.applyMatrix4(new Matrix4().makeScale(10, 10, 10));
-    this.p0 = new Vector3();
-    this.p1 = new Vector3();
-    this.p2 = new Vector3();
-    this.triangle = new Triangle();
-  }
-
-  setUvs() {
-    for (let j = 0; j < 468; j++) {
-      this.uvs[j * 2] = this.texCoords[j][0];
-      this.uvs[j * 2 + 1] = 1 - this.texCoords[j][1];
-    }
-    this.getAttribute('uv').needsUpdate = true;
-  }
-
-  setVideoUvs() {
-    let ptr = 0;
-    for (let j = 0; j < 468 * 2; j += 2) {
-      this.uvs[j] = 1 - (this.positions[ptr] / this.w + 0.5);
-      this.uvs[j + 1] = this.positions[ptr + 1] / this.h + 0.5;
-      ptr += 3;
-    }
-    this.getAttribute('uv').needsUpdate = true;
-  }
-
-  setSize(w, h) {
-    this.w = w;
-    this.h = h;
-  }
-
-  update(face) {
-    let ptr = 0;
-    for (const p of face.mesh) {
-      this.positions[ptr] = 1 - p[0] + 0.5 * this.w;
-      this.positions[ptr + 1] = this.h - p[1] - 0.5 * this.h;
-      this.positions[ptr + 2] = -p[2];
-      ptr += 3;
-    }
-    this.setVideoUvs();
-    this.attributes.position.needsUpdate = true;
-    this.computeVertexNormals();
-  }
-
-  track(id0, id1, id2) {
-    const points = this.positions;
-    this.p0.set(points[id0 * 3], points[id0 * 3 + 1], points[id0 * 3 + 2]);
-    this.p1.set(points[id1 * 3], points[id1 * 3 + 1], points[id1 * 3 + 2]);
-    this.p2.set(points[id2 * 3], points[id2 * 3 + 1], points[id2 * 3 + 2]);
-    this.triangle.set(this.p0, this.p1, this.p2);
-    const center = new Vector3();
-    this.triangle.getMidpoint(center);
-    const normal = new Vector3();
-    this.triangle.getNormal(normal);
-    const matrix = new Matrix4();
-    const x = this.p1.clone().sub(this.p2).normalize();
-    const y = this.p1.clone().sub(this.p0).normalize();
-    const z = new Vector3().crossVectors(x, y);
-    const y2 = new Vector3().crossVectors(x, z).normalize();
-    const z2 = new Vector3().crossVectors(x, y2).normalize();
-    matrix.makeBasis(x, y2, z2);
-    return { position: center, normal, rotation: matrix };
-  }
-}
-
 const wireframe = true; // enable wireframe overlay
 
 const canvas = document.getElementById('canvas');
@@ -109,12 +32,11 @@ let height = 0;
 const renderer = new WebGLRenderer({ antialias: true, alpha: true, canvas });
 renderer.setClearColor(0x000000);
 renderer.outputEncoding = sRGBEncoding;
-const scene = new Scene();
+const camera = new OrthographicCamera();
-const camera = new OrthographicCamera(1, 1, 1, 1, -1000, 1000);
 const controls = new OrbitControls(camera, renderer.domElement); // pan&zoom controls
 controls.enabled = true;
 const materialWireFrame = new MeshBasicMaterial({ // create wireframe material
-  color: 0xff00ff,
+  color: 0xffaaaa,
   wireframe: true,
 });
 const materialFace = new MeshBasicMaterial({ // create material for mask
@@ -122,32 +44,65 @@ const materialFace = new MeshBasicMaterial({ // create material for mask
   map: null, // will be created when the video is ready.
   side: DoubleSide,
 });
-const faceGeometry = new FaceGeometry(human.faceTriangulation, human.faceUVMap); // create a new geometry helper
+
+class FaceGeometry extends BufferGeometry {
+  constructor(triangulation) {
+    super();
+    this.positions = new Float32Array(478 * 3);
+    this.uvs = new Float32Array(478 * 2);
+    this.setAttribute('position', new BufferAttribute(this.positions, 3));
+    this.setAttribute('uv', new BufferAttribute(this.uvs, 2));
+    this.setIndex(triangulation);
+  }
+
+  update(face) {
+    let ptr = 0;
+    for (const p of face.mesh) {
+      this.positions[ptr + 0] = -p[0] + width / 2;
+      this.positions[ptr + 1] = height - p[1] - height / 2;
+      this.positions[ptr + 2] = -p[2];
+      ptr += 3;
+    }
+    ptr = 0;
+    for (const p of face.meshRaw) {
+      this.uvs[ptr + 0] = 0 + p[0];
+      this.uvs[ptr + 1] = 1 - p[1];
+      ptr += 2;
+    }
+    materialFace.map.update(); // update textures from video
+    this.attributes.position.needsUpdate = true; // vertices
+    this.attributes.uv.needsUpdate = true; // textures
+    this.computeVertexNormals();
+  }
+}
+
+const scene = new Scene();
+const faceGeometry = new FaceGeometry(human.faceTriangulation); // create a new geometry helper
 const mesh = new Mesh(faceGeometry, materialFace); // create mask mesh
 scene.add(mesh);
 
 function resize(input) {
   width = input.videoWidth;
   height = input.videoHeight;
-  camera.left = -0.5 * width;
+  camera.left = -width / 2;
-  camera.right = 0.5 * height;
+  camera.right = width / 2;
-  camera.top = 0.5 * width;
+  camera.top = height / 2;
-  camera.bottom = -0.5 * height;
+  camera.bottom = -height / 2;
+  camera.near = -100;
+  camera.far = 100;
+  camera.zoom = 3;
   camera.updateProjectionMatrix();
-  const videoAspectRatio = width / height;
+  renderer.setSize(width, height);
-  const windowAspectRatio = window.innerWidth / window.innerHeight;
-  const adjustedWidth = videoAspectRatio > windowAspectRatio ? window.innerWidth : window.innerHeight * videoAspectRatio;
-  const adjustedHeight = videoAspectRatio > windowAspectRatio ? window.innerWidth / videoAspectRatio : window.innerHeight;
-  renderer.setSize(adjustedWidth, adjustedHeight);
-  faceGeometry.setSize(width, height);
 }
 
+const isLive = (input) => input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
+
 async function render(input) {
-  const live = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused);
+  if (isLive(input)) {
-  if (!live) return;
     if (width !== input.videoWidth || height !== input.videoHeight) resize(input); // resize orthographic camera to video dimensions if necessary
     const res = await human.detect(input);
-  if (res?.face?.length > 0) faceGeometry.update(res.face[0]);
+    if (res?.face?.length > 0) {
+      faceGeometry.update(res.face[0]);
       // render the mask
       mesh.material = materialFace;
       renderer.autoClear = true;
@@ -157,19 +112,25 @@ async function render(input) {
         renderer.autoClear = false;
         renderer.render(scene, camera);
       }
+    }
+  }
   requestAnimationFrame(() => render(input));
 }
 
 // setup webcam
 async function setupCamera() {
-  const video = document.getElementById('video');
   if (!navigator.mediaDevices) return null;
+  const video = document.getElementById('video');
+  canvas.addEventListener('click', () => {
+    if (isLive(video)) video.pause();
+    else video.play();
+  });
   const constraints = {
     audio: false,
     video: { facingMode: 'user', resizeMode: 'crop-and-scale' },
   };
   if (window.innerWidth > window.innerHeight) constraints.video.width = { ideal: window.innerWidth };
-  else constraints.video.height = { ideal: (window.innerHeight - document.getElementById('menubar').offsetHeight) };
+  else constraints.video.height = { ideal: window.innerHeight };
   const stream = await navigator.mediaDevices.getUserMedia(constraints);
   if (stream) video.srcObject = stream;
   else return null;
@@ -177,19 +138,22 @@ async function setupCamera() {
     video.onloadeddata = async () => {
       video.width = video.videoWidth;
       video.height = video.videoHeight;
+      canvas.width = video.width;
+      canvas.height = video.height;
       video.play();
       resolve(video);
     };
   });
 }
 
-async function init() {
+async function main() {
-  // await av.ready();
   const video = await setupCamera();
-  const videoTexture = new VideoTexture(video); // load textures from video
+  if (video) {
+    const videoTexture = new VideoTexture(video); // now load textures from video
     videoTexture.encoding = sRGBEncoding;
     materialFace.map = videoTexture;
     render(video);
   }
+}
 
-window.onload = init;
+window.onload = main;

package.json

@@ -62,7 +62,7 @@
     "@vladmandic/pilogger": "^0.2.15",
     "chokidar": "^3.5.1",
     "dayjs": "^1.10.4",
-    "esbuild": "^0.11.0",
+    "esbuild": "^0.11.2",
     "eslint": "^7.23.0",
     "eslint-config-airbnb-base": "^14.2.1",
     "eslint-plugin-import": "^2.22.1",

src/image/image.ts

@@ -1,6 +1,5 @@
 // @ts-nocheck
 
-import { log } from '../helpers';
 import * as tf from '../../dist/tfjs.esm.js';
 import * as fxImage from './imagefx';
 
@@ -17,6 +16,9 @@ let fx = null;
 export function process(input, config): { tensor: tf.Tensor, canvas: OffscreenCanvas | HTMLCanvasElement } {
   let tensor;
   if (!input) throw new Error('Human: Input is missing');
+  if (!(input instanceof tf.Tensor) && !(input instanceof ImageData) && !(input instanceof ImageBitmap) && !(input instanceof HTMLVideoElement) && !(input instanceof HTMLCanvasElement) && !(input instanceof OffscreenCanvas)) {
+    throw new Error('Human: Input type is not recognized');
+  }
   if (input instanceof tf.Tensor) {
     tensor = tf.clone(input);
   } else {
@@ -36,10 +38,7 @@ export function process(input, config): { tensor: tf.Tensor, canvas: OffscreenCa
     else if (config.filter.height > 0) targetWidth = originalWidth * (config.filter.height / originalHeight);
     if (config.filter.height > 0) targetHeight = config.filter.height;
     else if (config.filter.width > 0) targetHeight = originalHeight * (config.filter.width / originalWidth);
-    if (!targetWidth || !targetHeight) {
+    if (!targetWidth || !targetHeight) throw new Error('Human: Input cannot determine dimension');
-      log('Human: invalid input', input);
-      return { tensor: null, canvas: null };
-    }
     if (!inCanvas || (inCanvas.width !== targetWidth) || (inCanvas.height !== targetHeight)) {
       inCanvas = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(targetWidth, targetHeight) : document.createElement('canvas');
       if (inCanvas.width !== targetWidth) inCanvas.width = targetWidth;

typedoc/index.html

@@ -1,218 +0,0 @@
-<!doctype html>
-<html class="default no-js">
-<head>
-	<meta charset="utf-8">
-	<meta http-equiv="X-UA-Compatible" content="IE=edge">
-	<title>@vladmandic/human</title>
-	<meta name="description" content="Documentation for @vladmandic/human">
-	<meta name="viewport" content="width=device-width, initial-scale=1">
-	<link rel="stylesheet" href="assets/css/main.css">
-	<script async src="assets/js/search.js" id="search-script"></script>
-</head>
-<body>
-<header>
-	<div class="tsd-page-toolbar">
-		<div class="container">
-			<div class="table-wrap">
-				<div class="table-cell" id="tsd-search" data-index="assets/js/search.json" data-base=".">
-					<div class="field">
-						<label for="tsd-search-field" class="tsd-widget search no-caption">Search</label>
-						<input id="tsd-search-field" type="text" />
-					</div>
-					<ul class="results">
-						<li class="state loading">Preparing search index...</li>
-						<li class="state failure">The search index is not available</li>
-					</ul>
-					<a href="index.html" class="title">@vladmandic/human</a>
-				</div>
-				<div class="table-cell" id="tsd-widgets">
-					<div id="tsd-filter">
-						<a href="#" class="tsd-widget options no-caption" data-toggle="options">Options</a>
-						<div class="tsd-filter-group">
-							<div class="tsd-select" id="tsd-filter-visibility">
-								<span class="tsd-select-label">All</span>
-								<ul class="tsd-select-list">
-									<li data-value="public">Public</li>
-									<li data-value="protected">Public/Protected</li>
-									<li data-value="private" class="selected">All</li>
-								</ul>
-							</div>
-							<input type="checkbox" id="tsd-filter-inherited" checked />
-							<label class="tsd-widget" for="tsd-filter-inherited">Inherited</label>
-						</div>
-					</div>
-					<a href="#" class="tsd-widget menu no-caption" data-toggle="menu">Menu</a>
-				</div>
-			</div>
-		</div>
-	</div>
-	<div class="tsd-page-title">
-		<div class="container">
-			<h1>@vladmandic/human</h1>
-		</div>
-	</div>
-</header>
-<div class="container container-main">
-	<div class="row">
-		<div class="col-8 col-content">
-			<section class="tsd-panel-group tsd-index-group">
-				<h2>Index</h2>
-				<section class="tsd-panel tsd-index-panel">
-					<div class="tsd-index-content">
-						<section class="tsd-index-section ">
-							<h3>References</h3>
-							<ul class="tsd-index-list">
-								<li class="tsd-kind-reference"><a href="index.html#default" class="tsd-kind-icon">default</a></li>
-							</ul>
-						</section>
-						<section class="tsd-index-section ">
-							<h3>Classes</h3>
-							<ul class="tsd-index-list">
-								<li class="tsd-kind-class"><a href="classes/human.html" class="tsd-kind-icon">Human</a></li>
-							</ul>
-						</section>
-						<section class="tsd-index-section ">
-							<h3>Interfaces</h3>
-							<ul class="tsd-index-list">
-								<li class="tsd-kind-interface"><a href="interfaces/config.html" class="tsd-kind-icon">Config</a></li>
-								<li class="tsd-kind-interface"><a href="interfaces/result.html" class="tsd-kind-icon">Result</a></li>
-							</ul>
-						</section>
-						<section class="tsd-index-section ">
-							<h3>Type aliases</h3>
-							<ul class="tsd-index-list">
-								<li class="tsd-kind-type-alias"><a href="index.html#error" class="tsd-kind-icon">Error</a></li>
-								<li class="tsd-kind-type-alias"><a href="index.html#input" class="tsd-kind-icon">Input</a></li>
-								<li class="tsd-kind-type-alias"><a href="index.html#tensor" class="tsd-kind-icon">Tensor</a></li>
-								<li class="tsd-kind-type-alias"><a href="index.html#tensorflow" class="tsd-kind-icon">Tensor<wbr>Flow</a></li>
-							</ul>
-						</section>
-					</div>
-				</section>
-			</section>
-			<section class="tsd-panel-group tsd-member-group ">
-				<h2>References</h2>
-				<section class="tsd-panel tsd-member tsd-kind-reference">
-					<a name="default" class="tsd-anchor"></a>
-					<h3>default</h3>
-					Renames and exports <a href="classes/human.html">Human</a>
-				</section>
-			</section>
-			<section class="tsd-panel-group tsd-member-group ">
-				<h2>Type aliases</h2>
-				<section class="tsd-panel tsd-member tsd-kind-type-alias">
-					<a name="error" class="tsd-anchor"></a>
-					<h3>Error</h3>
-					<div class="tsd-signature tsd-kind-icon">Error<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">{ </span>error<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span><span class="tsd-signature-symbol"> }</span></div>
-					<aside class="tsd-sources">
-					</aside>
-					<div class="tsd-comment tsd-typography">
-						<div class="lead">
-							<p>Error message</p>
-						</div>
-					</div>
-					<div class="tsd-type-declaration">
-						<h4>Type declaration</h4>
-						<ul class="tsd-parameters">
-							<li class="tsd-parameter">
-								<h5>error<span class="tsd-signature-symbol">: </span><span class="tsd-signature-type">string</span></h5>
-							</li>
-						</ul>
-					</div>
-				</section>
-				<section class="tsd-panel tsd-member tsd-kind-type-alias">
-					<a name="input" class="tsd-anchor"></a>
-					<h3>Input</h3>
-					<div class="tsd-signature tsd-kind-icon">Input<span class="tsd-signature-symbol">:</span> <a href="index.html#tensor" class="tsd-signature-type" data-tsd-kind="Type alias">Tensor</a><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">ImageData</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">ImageBitmap</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HTMLVideoElement</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">HTMLCanvasElement</span><span class="tsd-signature-symbol"> | </span><span class="tsd-signature-type">OffscreenCanvas</span></div>
-					<aside class="tsd-sources">
-					</aside>
-					<div class="tsd-comment tsd-typography">
-						<div class="lead">
-							<p>Defines all possible input types for <strong>Human</strong> detection</p>
-						</div>
-					</div>
-				</section>
-				<section class="tsd-panel tsd-member tsd-kind-type-alias">
-					<a name="tensor" class="tsd-anchor"></a>
-					<h3>Tensor</h3>
-					<div class="tsd-signature tsd-kind-icon">Tensor<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">typeof </span><span class="tsd-signature-type">tf.Tensor</span></div>
-					<aside class="tsd-sources">
-					</aside>
-					<div class="tsd-comment tsd-typography">
-						<div class="lead">
-							<p>Generic Tensor object type</p>
-						</div>
-					</div>
-				</section>
-				<section class="tsd-panel tsd-member tsd-kind-type-alias">
-					<a name="tensorflow" class="tsd-anchor"></a>
-					<h3>Tensor<wbr>Flow</h3>
-					<div class="tsd-signature tsd-kind-icon">Tensor<wbr>Flow<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-symbol">typeof </span><span class="tsd-signature-type">tf</span></div>
-					<aside class="tsd-sources">
-					</aside>
-					<div class="tsd-comment tsd-typography">
-						<div class="lead">
-							<p>Instance of TensorFlow/JS</p>
-						</div>
-					</div>
-				</section>
-			</section>
-		</div>
-		<div class="col-4 col-menu menu-sticky-wrap menu-highlight">
-			<nav class="tsd-navigation primary">
-				<ul>
-					<li class="current ">
-						<a href="index.html">Exports</a>
-					</li>
-				</ul>
-			</nav>
-			<nav class="tsd-navigation secondary menu-sticky">
-				<ul class="before-current">
-					<li class=" tsd-kind-reference">
-						<a href="index.html#default" class="tsd-kind-icon">default</a>
-					</li>
-					<li class=" tsd-kind-class">
-						<a href="classes/human.html" class="tsd-kind-icon">Human</a>
-					</li>
-					<li class=" tsd-kind-interface">
-						<a href="interfaces/config.html" class="tsd-kind-icon">Config</a>
-					</li>
-					<li class=" tsd-kind-interface">
-						<a href="interfaces/result.html" class="tsd-kind-icon">Result</a>
-					</li>
-					<li class=" tsd-kind-type-alias">
-						<a href="index.html#error" class="tsd-kind-icon">Error</a>
-					</li>
-					<li class=" tsd-kind-type-alias">
-						<a href="index.html#input" class="tsd-kind-icon">Input</a>
-					</li>
-					<li class=" tsd-kind-type-alias">
-						<a href="index.html#tensor" class="tsd-kind-icon">Tensor</a>
-					</li>
-					<li class=" tsd-kind-type-alias">
-						<a href="index.html#tensorflow" class="tsd-kind-icon">Tensor<wbr>Flow</a>
-					</li>
-				</ul>
-			</nav>
-		</div>
-	</div>
-</div>
-<footer>
-	<div class="container">
-		<h2>Legend</h2>
-		<div class="tsd-legend-group">
-			<ul class="tsd-legend">
-				<li class="tsd-kind-constructor tsd-parent-kind-class"><span class="tsd-kind-icon">Constructor</span></li>
-				<li class="tsd-kind-property tsd-parent-kind-class"><span class="tsd-kind-icon">Property</span></li>
-				<li class="tsd-kind-method tsd-parent-kind-class"><span class="tsd-kind-icon">Method</span></li>
-			</ul>
-			<ul class="tsd-legend">
-				<li class="tsd-kind-property tsd-parent-kind-interface"><span class="tsd-kind-icon">Property</span></li>
-			</ul>
-		</div>
-	</div>
-</footer>
-<div class="overlay"></div>
-<script src="assets/js/main.js"></script>
-</body>
-</html>

wiki

@@ -1 +1 @@
-Subproject commit df23c6e6fdd3aedd5da6ee08d90d61983a761e91
+Subproject commit 8bca078ef07d67c623bd2cb1a207a4bc8d743a60