blazeface optimizations

TODO.md

@@ -18,4 +18,4 @@ N/A
 
 ## In Progress
 
-N/A
+- Refactor BlazeFace to avoid data and array access and use buffers where needed

demo/index.js

@@ -6,21 +6,21 @@ import Menu from './helpers/menu.js';
 import GLBench from './helpers/gl-bench.js';
 import webRTC from './helpers/webrtc.js';
 
-const userConfig = { warmup: 'none' };
 let human;
 
-/*
 const userConfig = {
+  warmup: 'none',
+  /*
   backend: 'humangl',
   async: false,
   profile: false,
-  warmup: 'full',
+  videoOptimized: true,
-  videoOptimized: false,
   filter: {
     enabled: false,
     flip: false,
   },
   face: { enabled: true,
+    detector: { return: false },
     mesh: { enabled: true },
     iris: { enabled: true },
     description: { enabled: false },
@@ -31,8 +31,8 @@ const userConfig = {
   body: { enabled: false, modelPath: 'posenet.json' },
   // body: { enabled: true, modelPath: 'blazepose.json' },
   // object: { enabled: true },
+  */
 };
-*/
 
 // ui options
 const ui = {

dist/human.node-gpu.js

@@ -44,22 +44,26 @@ var __privateSet = (obj, member, value, setter) => {
 
 // dist/tfjs.esm.js
 var require_tfjs_esm = __commonJS((exports) => {
-  var s = Object.create;
+  var __create2 = Object.create;
-  var t = Object.defineProperty;
+  var __defProp2 = Object.defineProperty;
-  var d = Object.getPrototypeOf;
+  var __getProtoOf2 = Object.getPrototypeOf;
-  var g = Object.prototype.hasOwnProperty;
+  var __hasOwnProp2 = Object.prototype.hasOwnProperty;
-  var j = Object.getOwnPropertyNames;
+  var __getOwnPropNames2 = Object.getOwnPropertyNames;
-  var l = Object.getOwnPropertyDescriptor;
+  var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
-  var p = (o) => t(o, "__esModule", {value: true});
+  var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
-  var r = (o, e, n) => {
+  var __reExport2 = (target, module22, desc) => {
-    if (e && typeof e == "object" || typeof e == "function")
+    if (module22 && typeof module22 === "object" || typeof module22 === "function") {
-      for (let f of j(e))
+      for (let key of __getOwnPropNames2(module22))
-        !g.call(o, f) && f !== "default" && t(o, f, {get: () => e[f], enumerable: !(n = l(e, f)) || n.enumerable});
+        if (!__hasOwnProp2.call(target, key) && key !== "default")
-    return o;
+          __defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
+    }
+    return target;
   };
-  var m = (o) => r(p(t(o != null ? s(d(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
+  var __toModule2 = (module22) => {
-  p(exports);
+    return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
-  r(exports, m(require("@tensorflow/tfjs-node-gpu")));
+  };
+  __markAsModule2(exports);
+  __reExport2(exports, __toModule2(require("@tensorflow/tfjs-node-gpu")));
 });
 
 // src/human.ts
@@ -254,8 +258,8 @@ function squarifyBox(box4) {
   const size = getBoxSize(box4);
   const maxEdge = Math.max(...size);
   const halfSize = maxEdge / 2;
-  const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
+  const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
-  const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
+  const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
   return {startPoint, endPoint, landmarks: box4.landmarks};
 }
 function calculateLandmarksBoundingBox(landmarks) {
@@ -266,7 +270,6 @@ function calculateLandmarksBoundingBox(landmarks) {
   return {startPoint, endPoint, landmarks};
 }
 var createBox = (startEndTensor) => ({
-  startEndTensor,
   startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
   endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
 });
@@ -387,46 +390,39 @@ var BlazeFaceModel = class {
     const [batch, boxes, scores] = tf3.tidy(() => {
       const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
       const normalizedImage = resizedImage.div(127.5).sub(0.5);
-      const batchedPrediction = this.model.predict(normalizedImage);
+      const res = this.model.execute(normalizedImage);
       let batchOut;
-      if (Array.isArray(batchedPrediction)) {
+      if (Array.isArray(res)) {
-        const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
+        const sorted = res.sort((a, b) => a.size - b.size);
         const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
         const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
         const concat3 = tf3.concat([concat512, concat384], 1);
         batchOut = concat3.squeeze(0);
       } else {
-        batchOut = batchedPrediction.squeeze();
+        batchOut = res.squeeze();
       }
       const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
       const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
-      const scoresOut = tf3.sigmoid(logits).squeeze();
+      const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
       return [batchOut, boxesOut, scoresOut];
     });
-    const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
+    const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
-    const boxIndices = boxIndicesTensor.arraySync();
+    const nms = nmsTensor.arraySync();
-    boxIndicesTensor.dispose();
+    nmsTensor.dispose();
-    const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
-    const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
-      const vals = boundingBox.arraySync();
-      boundingBox.dispose();
-      return vals;
-    });
-    const scoresVal = scores.dataSync();
     const annotatedBoxes = [];
-    for (let i = 0; i < boundingBoxes.length; i++) {
+    for (let i = 0; i < nms.length; i++) {
-      const boxIndex = boxIndices[i];
+      const confidence = scores[nms[i]];
-      const confidence = scoresVal[boxIndex];
       if (confidence > this.config.face.detector.minConfidence) {
-        const localBox = createBox(boundingBoxes[i]);
+        const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
-        const anchor = this.anchorsData[boxIndex];
+        const localBox = createBox(boundingBox);
-        const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
+        boundingBox.dispose();
+        const anchor = this.anchorsData[nms[i]];
+        const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
         annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
       }
     }
     batch.dispose();
     boxes.dispose();
-    scores.dispose();
     return {
       boxes: annotatedBoxes,
       scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@@ -3784,9 +3780,9 @@ var Pipeline = class {
     const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
     const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
     return coordsRotated.map((coord) => [
-      coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
+      Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
-      coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
+      Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
-      coord[2]
+      Math.round(coord[2])
     ]);
   }
   getLeftToRightEyeDepthDifference(rawCoords) {
@@ -3878,8 +3874,7 @@ var Pipeline = class {
         prediction.landmarks.dispose();
       });
     }
-    let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
+    const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
-      const boxConfidence = box4.confidence;
       let face4;
       let angle = 0;
       let rotationMatrix;
@@ -3904,19 +3899,21 @@ var Pipeline = class {
       }
       if (!config3.face.mesh.enabled) {
         const prediction2 = {
-          coords: null,
+          mesh: [],
           box: box4,
           faceConfidence: null,
-          boxConfidence,
+          boxConfidence: box4.confidence,
           confidence: box4.confidence,
           image: face4
         };
         return prediction2;
       }
-      const [, confidence, contourCoords] = this.meshDetector.predict(face4);
+      const [, confidence, contourCoords] = this.meshDetector.execute(face4);
       const faceConfidence = confidence.dataSync()[0];
-      if (faceConfidence < config3.face.detector.minConfidence)
+      if (faceConfidence < config3.face.detector.minConfidence) {
+        this.storedBoxes[i].confidence = faceConfidence;
         return null;
+      }
       const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
       let rawCoords = coordsReshaped.arraySync();
       if (config3.face.iris.enabled) {
@@ -3941,9 +3938,10 @@ var Pipeline = class {
         const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
         rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
       }
-      const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
+      const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
-      box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
+      const storeConfidence = box4.confidence;
-      const transformedCoords = tf4.tensor2d(transformedCoordsData);
+      box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
+      box4.confidence = storeConfidence;
       if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
         const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
         angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
@@ -3954,20 +3952,20 @@ var Pipeline = class {
         face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
       }
       const prediction = {
-        coords: transformedCoords,
+        mesh,
         box: box4,
         faceConfidence,
-        boxConfidence,
+        boxConfidence: box4.confidence,
-        image: face4,
+        image: face4
-        rawCoords
       };
-      const squarifiedLandmarksBox = squarifyBox(box4);
+      const storedBox = squarifyBox(box4);
-      this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
+      storedBox.confidence = box4.confidence;
+      storedBox.faceConfidence = faceConfidence;
+      this.storedBoxes[i] = storedBox;
       return prediction;
     }));
-    results = results.filter((a) => a !== null);
     if (config3.face.mesh.enabled)
-      this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
+      this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
     this.detectedFaces = results.length;
     return results;
   }
@@ -3980,20 +3978,19 @@ async function predict(input, config3) {
   const predictions = await facePipeline.predict(input, config3);
   const results = [];
   for (const prediction of predictions || []) {
-    if (prediction.isDisposedInternal)
+    if (!prediction || prediction.isDisposedInternal)
       continue;
-    const mesh = prediction.coords ? prediction.coords.arraySync() : [];
+    const meshRaw = prediction.mesh.map((pt) => [
-    const meshRaw = mesh.map((pt) => [
       pt[0] / input.shape[2],
       pt[1] / input.shape[1],
       pt[2] / facePipeline.meshSize
     ]);
     const annotations3 = {};
-    if (mesh && mesh.length > 0) {
+    if (prediction.mesh && prediction.mesh.length > 0) {
       for (const key of Object.keys(MESH_ANNOTATIONS))
-        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
+        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
     }
-    const box4 = prediction.box ? [
+    const clampedBox = prediction.box ? [
       Math.max(0, prediction.box.startPoint[0]),
       Math.max(0, prediction.box.startPoint[1]),
       Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
@@ -4009,17 +4006,15 @@ async function predict(input, config3) {
       confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
       boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
       faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
-      box: box4,
+      box: clampedBox,
       boxRaw,
-      mesh,
+      mesh: prediction.mesh,
       meshRaw,
       annotations: annotations3,
-      image: prediction.image ? prediction.image.clone() : null
+      image: prediction.image
     });
     if (prediction.coords)
       prediction.coords.dispose();
-    if (prediction.image)
-      prediction.image.dispose();
   }
   return results;
 }

dist/human.node-wasm.js

@@ -44,23 +44,27 @@ var __privateSet = (obj, member, value, setter) => {
 
 // dist/tfjs.esm.js
 var require_tfjs_esm = __commonJS((exports) => {
-  var w = Object.create;
+  var __create2 = Object.create;
-  var e = Object.defineProperty;
+  var __defProp2 = Object.defineProperty;
-  var a = Object.getPrototypeOf;
+  var __getProtoOf2 = Object.getPrototypeOf;
-  var j = Object.prototype.hasOwnProperty;
+  var __hasOwnProp2 = Object.prototype.hasOwnProperty;
-  var l = Object.getOwnPropertyNames;
+  var __getOwnPropNames2 = Object.getOwnPropertyNames;
-  var p = Object.getOwnPropertyDescriptor;
+  var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
-  var m = (o) => e(o, "__esModule", {value: true});
+  var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
-  var f = (o, r, s) => {
+  var __reExport2 = (target, module22, desc) => {
-    if (r && typeof r == "object" || typeof r == "function")
+    if (module22 && typeof module22 === "object" || typeof module22 === "function") {
-      for (let t of l(r))
+      for (let key of __getOwnPropNames2(module22))
-        !j.call(o, t) && t !== "default" && e(o, t, {get: () => r[t], enumerable: !(s = p(r, t)) || s.enumerable});
+        if (!__hasOwnProp2.call(target, key) && key !== "default")
-    return o;
+          __defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
+    }
+    return target;
   };
-  var n = (o) => f(m(e(o != null ? w(a(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
+  var __toModule2 = (module22) => {
-  m(exports);
+    return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
-  f(exports, n(require("@tensorflow/tfjs")));
+  };
-  f(exports, n(require("@tensorflow/tfjs-backend-wasm")));
+  __markAsModule2(exports);
+  __reExport2(exports, __toModule2(require("@tensorflow/tfjs")));
+  __reExport2(exports, __toModule2(require("@tensorflow/tfjs-backend-wasm")));
 });
 
 // src/human.ts
@@ -255,8 +259,8 @@ function squarifyBox(box4) {
   const size = getBoxSize(box4);
   const maxEdge = Math.max(...size);
   const halfSize = maxEdge / 2;
-  const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
+  const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
-  const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
+  const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
   return {startPoint, endPoint, landmarks: box4.landmarks};
 }
 function calculateLandmarksBoundingBox(landmarks) {
@@ -267,7 +271,6 @@ function calculateLandmarksBoundingBox(landmarks) {
   return {startPoint, endPoint, landmarks};
 }
 var createBox = (startEndTensor) => ({
-  startEndTensor,
   startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
   endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
 });
@@ -388,46 +391,39 @@ var BlazeFaceModel = class {
     const [batch, boxes, scores] = tf3.tidy(() => {
       const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
       const normalizedImage = resizedImage.div(127.5).sub(0.5);
-      const batchedPrediction = this.model.predict(normalizedImage);
+      const res = this.model.execute(normalizedImage);
       let batchOut;
-      if (Array.isArray(batchedPrediction)) {
+      if (Array.isArray(res)) {
-        const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
+        const sorted = res.sort((a, b) => a.size - b.size);
         const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
         const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
         const concat3 = tf3.concat([concat512, concat384], 1);
         batchOut = concat3.squeeze(0);
       } else {
-        batchOut = batchedPrediction.squeeze();
+        batchOut = res.squeeze();
       }
       const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
       const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
-      const scoresOut = tf3.sigmoid(logits).squeeze();
+      const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
       return [batchOut, boxesOut, scoresOut];
     });
-    const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
+    const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
-    const boxIndices = boxIndicesTensor.arraySync();
+    const nms = nmsTensor.arraySync();
-    boxIndicesTensor.dispose();
+    nmsTensor.dispose();
-    const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
-    const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
-      const vals = boundingBox.arraySync();
-      boundingBox.dispose();
-      return vals;
-    });
-    const scoresVal = scores.dataSync();
     const annotatedBoxes = [];
-    for (let i = 0; i < boundingBoxes.length; i++) {
+    for (let i = 0; i < nms.length; i++) {
-      const boxIndex = boxIndices[i];
+      const confidence = scores[nms[i]];
-      const confidence = scoresVal[boxIndex];
       if (confidence > this.config.face.detector.minConfidence) {
-        const localBox = createBox(boundingBoxes[i]);
+        const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
-        const anchor = this.anchorsData[boxIndex];
+        const localBox = createBox(boundingBox);
-        const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
+        boundingBox.dispose();
+        const anchor = this.anchorsData[nms[i]];
+        const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
         annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
       }
     }
     batch.dispose();
     boxes.dispose();
-    scores.dispose();
     return {
       boxes: annotatedBoxes,
       scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@@ -3785,9 +3781,9 @@ var Pipeline = class {
     const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
     const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
     return coordsRotated.map((coord) => [
-      coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
+      Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
-      coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
+      Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
-      coord[2]
+      Math.round(coord[2])
     ]);
   }
   getLeftToRightEyeDepthDifference(rawCoords) {
@@ -3879,8 +3875,7 @@ var Pipeline = class {
         prediction.landmarks.dispose();
       });
     }
-    let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
+    const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
-      const boxConfidence = box4.confidence;
       let face4;
       let angle = 0;
       let rotationMatrix;
@@ -3905,19 +3900,21 @@ var Pipeline = class {
       }
       if (!config3.face.mesh.enabled) {
         const prediction2 = {
-          coords: null,
+          mesh: [],
           box: box4,
           faceConfidence: null,
-          boxConfidence,
+          boxConfidence: box4.confidence,
           confidence: box4.confidence,
           image: face4
         };
         return prediction2;
       }
-      const [, confidence, contourCoords] = this.meshDetector.predict(face4);
+      const [, confidence, contourCoords] = this.meshDetector.execute(face4);
       const faceConfidence = confidence.dataSync()[0];
-      if (faceConfidence < config3.face.detector.minConfidence)
+      if (faceConfidence < config3.face.detector.minConfidence) {
+        this.storedBoxes[i].confidence = faceConfidence;
         return null;
+      }
       const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
       let rawCoords = coordsReshaped.arraySync();
       if (config3.face.iris.enabled) {
@@ -3942,9 +3939,10 @@ var Pipeline = class {
         const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
         rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
       }
-      const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
+      const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
-      box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
+      const storeConfidence = box4.confidence;
-      const transformedCoords = tf4.tensor2d(transformedCoordsData);
+      box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
+      box4.confidence = storeConfidence;
       if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
         const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
         angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
@@ -3955,20 +3953,20 @@ var Pipeline = class {
         face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
       }
       const prediction = {
-        coords: transformedCoords,
+        mesh,
         box: box4,
         faceConfidence,
-        boxConfidence,
+        boxConfidence: box4.confidence,
-        image: face4,
+        image: face4
-        rawCoords
       };
-      const squarifiedLandmarksBox = squarifyBox(box4);
+      const storedBox = squarifyBox(box4);
-      this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
+      storedBox.confidence = box4.confidence;
+      storedBox.faceConfidence = faceConfidence;
+      this.storedBoxes[i] = storedBox;
       return prediction;
     }));
-    results = results.filter((a) => a !== null);
     if (config3.face.mesh.enabled)
-      this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
+      this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
     this.detectedFaces = results.length;
     return results;
   }
@@ -3981,20 +3979,19 @@ async function predict(input, config3) {
   const predictions = await facePipeline.predict(input, config3);
   const results = [];
   for (const prediction of predictions || []) {
-    if (prediction.isDisposedInternal)
+    if (!prediction || prediction.isDisposedInternal)
       continue;
-    const mesh = prediction.coords ? prediction.coords.arraySync() : [];
+    const meshRaw = prediction.mesh.map((pt) => [
-    const meshRaw = mesh.map((pt) => [
       pt[0] / input.shape[2],
       pt[1] / input.shape[1],
       pt[2] / facePipeline.meshSize
     ]);
     const annotations3 = {};
-    if (mesh && mesh.length > 0) {
+    if (prediction.mesh && prediction.mesh.length > 0) {
       for (const key of Object.keys(MESH_ANNOTATIONS))
-        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
+        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
     }
-    const box4 = prediction.box ? [
+    const clampedBox = prediction.box ? [
       Math.max(0, prediction.box.startPoint[0]),
       Math.max(0, prediction.box.startPoint[1]),
       Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
@@ -4010,17 +4007,15 @@ async function predict(input, config3) {
       confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
       boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
       faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
-      box: box4,
+      box: clampedBox,
       boxRaw,
-      mesh,
+      mesh: prediction.mesh,
       meshRaw,
       annotations: annotations3,
-      image: prediction.image ? prediction.image.clone() : null
+      image: prediction.image
     });
     if (prediction.coords)
       prediction.coords.dispose();
-    if (prediction.image)
-      prediction.image.dispose();
   }
   return results;
 }

dist/human.node.js

@@ -44,22 +44,26 @@ var __privateSet = (obj, member, value, setter) => {
 
 // dist/tfjs.esm.js
 var require_tfjs_esm = __commonJS((exports) => {
-  var d = Object.create;
+  var __create2 = Object.create;
-  var t = Object.defineProperty;
+  var __defProp2 = Object.defineProperty;
-  var j = Object.getPrototypeOf;
+  var __getProtoOf2 = Object.getPrototypeOf;
-  var l = Object.prototype.hasOwnProperty;
+  var __hasOwnProp2 = Object.prototype.hasOwnProperty;
-  var m = Object.getOwnPropertyNames;
+  var __getOwnPropNames2 = Object.getOwnPropertyNames;
-  var p = Object.getOwnPropertyDescriptor;
+  var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
-  var s = (o) => t(o, "__esModule", {value: true});
+  var __markAsModule2 = (target) => __defProp2(target, "__esModule", {value: true});
-  var r = (o, e, n) => {
+  var __reExport2 = (target, module22, desc) => {
-    if (e && typeof e == "object" || typeof e == "function")
+    if (module22 && typeof module22 === "object" || typeof module22 === "function") {
-      for (let f of m(e))
+      for (let key of __getOwnPropNames2(module22))
-        !l.call(o, f) && f !== "default" && t(o, f, {get: () => e[f], enumerable: !(n = p(e, f)) || n.enumerable});
+        if (!__hasOwnProp2.call(target, key) && key !== "default")
-    return o;
+          __defProp2(target, key, {get: () => module22[key], enumerable: !(desc = __getOwnPropDesc2(module22, key)) || desc.enumerable});
+    }
+    return target;
   };
-  var w = (o) => r(s(t(o != null ? d(j(o)) : {}, "default", o && o.__esModule && "default" in o ? {get: () => o.default, enumerable: true} : {value: o, enumerable: true})), o);
+  var __toModule2 = (module22) => {
-  s(exports);
+    return __reExport2(__markAsModule2(__defProp2(module22 != null ? __create2(__getProtoOf2(module22)) : {}, "default", module22 && module22.__esModule && "default" in module22 ? {get: () => module22.default, enumerable: true} : {value: module22, enumerable: true})), module22);
-  r(exports, w(require("@tensorflow/tfjs-node")));
+  };
+  __markAsModule2(exports);
+  __reExport2(exports, __toModule2(require("@tensorflow/tfjs-node")));
 });
 
 // src/human.ts
@@ -254,8 +258,8 @@ function squarifyBox(box4) {
   const size = getBoxSize(box4);
   const maxEdge = Math.max(...size);
   const halfSize = maxEdge / 2;
-  const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
+  const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
-  const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
+  const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
   return {startPoint, endPoint, landmarks: box4.landmarks};
 }
 function calculateLandmarksBoundingBox(landmarks) {
@@ -266,7 +270,6 @@ function calculateLandmarksBoundingBox(landmarks) {
   return {startPoint, endPoint, landmarks};
 }
 var createBox = (startEndTensor) => ({
-  startEndTensor,
   startPoint: tf2.slice(startEndTensor, [0, 0], [-1, 2]),
   endPoint: tf2.slice(startEndTensor, [0, 2], [-1, 2])
 });
@@ -387,46 +390,39 @@ var BlazeFaceModel = class {
     const [batch, boxes, scores] = tf3.tidy(() => {
       const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
       const normalizedImage = resizedImage.div(127.5).sub(0.5);
-      const batchedPrediction = this.model.predict(normalizedImage);
+      const res = this.model.execute(normalizedImage);
       let batchOut;
-      if (Array.isArray(batchedPrediction)) {
+      if (Array.isArray(res)) {
-        const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
+        const sorted = res.sort((a, b) => a.size - b.size);
         const concat384 = tf3.concat([sorted[0], sorted[2]], 2);
         const concat512 = tf3.concat([sorted[1], sorted[3]], 2);
         const concat3 = tf3.concat([concat512, concat384], 1);
         batchOut = concat3.squeeze(0);
       } else {
-        batchOut = batchedPrediction.squeeze();
+        batchOut = res.squeeze();
       }
       const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
       const logits = tf3.slice(batchOut, [0, 0], [-1, 1]);
-      const scoresOut = tf3.sigmoid(logits).squeeze();
+      const scoresOut = tf3.sigmoid(logits).squeeze().dataSync();
       return [batchOut, boxesOut, scoresOut];
     });
-    const boxIndicesTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
+    const nmsTensor = await tf3.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
-    const boxIndices = boxIndicesTensor.arraySync();
+    const nms = nmsTensor.arraySync();
-    boxIndicesTensor.dispose();
+    nmsTensor.dispose();
-    const boundingBoxesMap = boxIndices.map((boxIndex) => tf3.slice(boxes, [boxIndex, 0], [1, -1]));
-    const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
-      const vals = boundingBox.arraySync();
-      boundingBox.dispose();
-      return vals;
-    });
-    const scoresVal = scores.dataSync();
     const annotatedBoxes = [];
-    for (let i = 0; i < boundingBoxes.length; i++) {
+    for (let i = 0; i < nms.length; i++) {
-      const boxIndex = boxIndices[i];
+      const confidence = scores[nms[i]];
-      const confidence = scoresVal[boxIndex];
       if (confidence > this.config.face.detector.minConfidence) {
-        const localBox = createBox(boundingBoxes[i]);
+        const boundingBox = tf3.slice(boxes, [nms[i], 0], [1, -1]);
-        const anchor = this.anchorsData[boxIndex];
+        const localBox = createBox(boundingBox);
-        const landmarks = tf3.tidy(() => tf3.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
+        boundingBox.dispose();
+        const anchor = this.anchorsData[nms[i]];
+        const landmarks = tf3.tidy(() => tf3.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
         annotatedBoxes.push({box: localBox, landmarks, anchor, confidence});
       }
     }
     batch.dispose();
     boxes.dispose();
-    scores.dispose();
     return {
       boxes: annotatedBoxes,
       scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize]
@@ -3784,9 +3780,9 @@ var Pipeline = class {
     const inverseRotationMatrix = angle !== 0 ? invertTransformMatrix(rotationMatrix) : IDENTITY_MATRIX;
     const boxCenter = [...getBoxCenter({startPoint: box4.startPoint, endPoint: box4.endPoint}), 1];
     return coordsRotated.map((coord) => [
-      coord[0] + dot(boxCenter, inverseRotationMatrix[0]),
+      Math.round(coord[0] + dot(boxCenter, inverseRotationMatrix[0])),
-      coord[1] + dot(boxCenter, inverseRotationMatrix[1]),
+      Math.round(coord[1] + dot(boxCenter, inverseRotationMatrix[1])),
-      coord[2]
+      Math.round(coord[2])
     ]);
   }
   getLeftToRightEyeDepthDifference(rawCoords) {
@@ -3878,8 +3874,7 @@ var Pipeline = class {
         prediction.landmarks.dispose();
       });
     }
-    let results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
+    const results = tf4.tidy(() => this.storedBoxes.map((box4, i) => {
-      const boxConfidence = box4.confidence;
       let face4;
       let angle = 0;
       let rotationMatrix;
@@ -3904,19 +3899,21 @@ var Pipeline = class {
       }
       if (!config3.face.mesh.enabled) {
         const prediction2 = {
-          coords: null,
+          mesh: [],
           box: box4,
           faceConfidence: null,
-          boxConfidence,
+          boxConfidence: box4.confidence,
           confidence: box4.confidence,
           image: face4
         };
         return prediction2;
       }
-      const [, confidence, contourCoords] = this.meshDetector.predict(face4);
+      const [, confidence, contourCoords] = this.meshDetector.execute(face4);
       const faceConfidence = confidence.dataSync()[0];
-      if (faceConfidence < config3.face.detector.minConfidence)
+      if (faceConfidence < config3.face.detector.minConfidence) {
+        this.storedBoxes[i].confidence = faceConfidence;
         return null;
+      }
       const coordsReshaped = tf4.reshape(contourCoords, [-1, 3]);
       let rawCoords = coordsReshaped.arraySync();
       if (config3.face.iris.enabled) {
@@ -3941,9 +3938,10 @@ var Pipeline = class {
         const adjustedRightIrisCoords = this.getAdjustedIrisCoords(rawCoords, rightIrisRawCoords, "right");
         rawCoords = rawCoords.concat(adjustedLeftIrisCoords).concat(adjustedRightIrisCoords);
       }
-      const transformedCoordsData = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
+      const mesh = this.transformRawCoords(rawCoords, box4, angle, rotationMatrix);
-      box4 = enlargeBox(calculateLandmarksBoundingBox(transformedCoordsData), 1.5);
+      const storeConfidence = box4.confidence;
-      const transformedCoords = tf4.tensor2d(transformedCoordsData);
+      box4 = enlargeBox(calculateLandmarksBoundingBox(mesh), 1.5);
+      box4.confidence = storeConfidence;
       if (config3.face.detector.rotation && config3.face.mesh.enabled && config3.face.description.enabled && tf4.ENV.flags.IS_BROWSER) {
         const [indexOfMouth, indexOfForehead] = box4.landmarks.length >= meshLandmarks.count ? meshLandmarks.symmetryLine : blazeFaceLandmarks.symmetryLine;
         angle = computeRotation(box4.landmarks[indexOfMouth], box4.landmarks[indexOfForehead]);
@@ -3954,20 +3952,20 @@ var Pipeline = class {
         face4 = cutBoxFromImageAndResize({startPoint: box4.startPoint, endPoint: box4.endPoint}, rotatedImage, [this.meshSize, this.meshSize]).div(255);
       }
       const prediction = {
-        coords: transformedCoords,
+        mesh,
         box: box4,
         faceConfidence,
-        boxConfidence,
+        boxConfidence: box4.confidence,
-        image: face4,
+        image: face4
-        rawCoords
       };
-      const squarifiedLandmarksBox = squarifyBox(box4);
+      const storedBox = squarifyBox(box4);
-      this.storedBoxes[i] = {...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box4.confidence, faceConfidence};
+      storedBox.confidence = box4.confidence;
+      storedBox.faceConfidence = faceConfidence;
+      this.storedBoxes[i] = storedBox;
       return prediction;
     }));
-    results = results.filter((a) => a !== null);
     if (config3.face.mesh.enabled)
-      this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config3.face.detector.minConfidence);
+      this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config3.face.detector.minConfidence);
     this.detectedFaces = results.length;
     return results;
   }
@@ -3980,20 +3978,19 @@ async function predict(input, config3) {
   const predictions = await facePipeline.predict(input, config3);
   const results = [];
   for (const prediction of predictions || []) {
-    if (prediction.isDisposedInternal)
+    if (!prediction || prediction.isDisposedInternal)
       continue;
-    const mesh = prediction.coords ? prediction.coords.arraySync() : [];
+    const meshRaw = prediction.mesh.map((pt) => [
-    const meshRaw = mesh.map((pt) => [
       pt[0] / input.shape[2],
       pt[1] / input.shape[1],
       pt[2] / facePipeline.meshSize
     ]);
     const annotations3 = {};
-    if (mesh && mesh.length > 0) {
+    if (prediction.mesh && prediction.mesh.length > 0) {
       for (const key of Object.keys(MESH_ANNOTATIONS))
-        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => mesh[index]);
+        annotations3[key] = MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
     }
-    const box4 = prediction.box ? [
+    const clampedBox = prediction.box ? [
       Math.max(0, prediction.box.startPoint[0]),
       Math.max(0, prediction.box.startPoint[1]),
       Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
@@ -4009,17 +4006,15 @@ async function predict(input, config3) {
       confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
       boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
       faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
-      box: box4,
+      box: clampedBox,
       boxRaw,
-      mesh,
+      mesh: prediction.mesh,
       meshRaw,
       annotations: annotations3,
-      image: prediction.image ? prediction.image.clone() : null
+      image: prediction.image
     });
     if (prediction.coords)
       prediction.coords.dispose();
-    if (prediction.image)
-      prediction.image.dispose();
   }
   return results;
 }

src/blazeface/blazeface.ts

@@ -40,50 +40,42 @@ export class BlazeFaceModel {
     if ((!inputImage) || (inputImage.isDisposedInternal) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return null;
     const [batch, boxes, scores] = tf.tidy(() => {
       const resizedImage = inputImage.resizeBilinear([this.inputSize, this.inputSize]);
-      // const normalizedImage = tf.mul(tf.sub(resizedImage.div(255), 0.5), 2);
       const normalizedImage = resizedImage.div(127.5).sub(0.5);
-      const batchedPrediction = this.model.predict(normalizedImage);
+      const res = this.model.execute(normalizedImage);
       let batchOut;
-      // are we using tfhub or pinto converted model?
+      if (Array.isArray(res)) { // are we using tfhub or pinto converted model?
-      if (Array.isArray(batchedPrediction)) {
+        const sorted = res.sort((a, b) => a.size - b.size);
-        const sorted = batchedPrediction.sort((a, b) => a.size - b.size);
         const concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
         const concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
         const concat = tf.concat([concat512, concat384], 1);
         batchOut = concat.squeeze(0);
       } else {
-        batchOut = batchedPrediction.squeeze(); // when using tfhub model
+        batchOut = res.squeeze(); // when using tfhub model
       }
       const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
       const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
-      const scoresOut = tf.sigmoid(logits).squeeze();
+      const scoresOut = tf.sigmoid(logits).squeeze().dataSync();
       return [batchOut, boxesOut, scoresOut];
     });
-    const boxIndicesTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
+    const nmsTensor = await tf.image.nonMaxSuppressionAsync(boxes, scores, this.config.face.detector.maxDetected, this.config.face.detector.iouThreshold, this.config.face.detector.minConfidence);
-    const boxIndices = boxIndicesTensor.arraySync();
+    const nms = nmsTensor.arraySync();
-    boxIndicesTensor.dispose();
+    nmsTensor.dispose();
-    const boundingBoxesMap = boxIndices.map((boxIndex) => tf.slice(boxes, [boxIndex, 0], [1, -1]));
+    const annotatedBoxes: Array<{ box: any, landmarks: any, anchor: number[], confidence: number }> = [];
-    const boundingBoxes = boundingBoxesMap.map((boundingBox) => {
+    for (let i = 0; i < nms.length; i++) {
-      const vals = boundingBox.arraySync();
+      const confidence = scores[nms[i]];
-      boundingBox.dispose();
-      return vals;
-    });
-
-    const scoresVal = scores.dataSync();
-    const annotatedBoxes: Array<{ box: any, landmarks: any, anchor: any, confidence: number }> = [];
-    for (let i = 0; i < boundingBoxes.length; i++) {
-      const boxIndex = boxIndices[i];
-      const confidence = scoresVal[boxIndex];
       if (confidence > this.config.face.detector.minConfidence) {
-        const localBox = box.createBox(boundingBoxes[i]);
+        const boundingBox = tf.slice(boxes, [nms[i], 0], [1, -1]);
-        const anchor = this.anchorsData[boxIndex];
+        const localBox = box.createBox(boundingBox);
-        const landmarks = tf.tidy(() => tf.slice(batch, [boxIndex, keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
+        boundingBox.dispose();
+        const anchor = this.anchorsData[nms[i]];
+        const landmarks = tf.tidy(() => tf.slice(batch, [nms[i], keypointsCount - 1], [1, -1]).squeeze().reshape([keypointsCount, -1]));
         annotatedBoxes.push({ box: localBox, landmarks, anchor, confidence });
       }
     }
+    // boundingBoxes.forEach((t) => t.dispose());
     batch.dispose();
     boxes.dispose();
-    scores.dispose();
+    // scores.dispose();
     return {
       boxes: annotatedBoxes,
       scaleFactor: [inputImage.shape[2] / this.inputSize, inputImage.shape[1] / this.inputSize],

src/blazeface/box.ts

@@ -46,8 +46,8 @@ export function squarifyBox(box) {
   const size = getBoxSize(box);
   const maxEdge = Math.max(...size);
   const halfSize = maxEdge / 2;
-  const startPoint = [centers[0] - halfSize, centers[1] - halfSize];
+  const startPoint = [Math.round(centers[0] - halfSize), Math.round(centers[1] - halfSize)];
-  const endPoint = [centers[0] + halfSize, centers[1] + halfSize];
+  const endPoint = [Math.round(centers[0] + halfSize), Math.round(centers[1] + halfSize)];
   return { startPoint, endPoint, landmarks: box.landmarks };
 }
 
@@ -60,13 +60,11 @@ export function calculateLandmarksBoundingBox(landmarks) {
 }
 
 export const disposeBox = (t) => {
-  t.startEndTensor.dispose();
   t.startPoint.dispose();
   t.endPoint.dispose();
 };
 
 export const createBox = (startEndTensor) => ({
-  startEndTensor,
   startPoint: tf.slice(startEndTensor, [0, 0], [-1, 2]),
   endPoint: tf.slice(startEndTensor, [0, 2], [-1, 2]),
 });

src/blazeface/facemesh.ts

@@ -11,18 +11,17 @@ export async function predict(input, config): Promise<{ confidence, boxConfidenc
   const predictions = await facePipeline.predict(input, config);
   const results: Array<{ confidence, boxConfidence, faceConfidence, box, mesh, boxRaw, meshRaw, annotations, image }> = [];
   for (const prediction of (predictions || [])) {
-    if (prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
+    if (!prediction || prediction.isDisposedInternal) continue; // guard against disposed tensors on long running operations such as pause in middle of processing
-    const mesh = prediction.coords ? prediction.coords.arraySync() : [];
+    const meshRaw = prediction.mesh.map((pt) => [
-    const meshRaw = mesh.map((pt) => [
       pt[0] / input.shape[2],
       pt[1] / input.shape[1],
       pt[2] / facePipeline.meshSize,
     ]);
     const annotations = {};
-    if (mesh && mesh.length > 0) {
+    if (prediction.mesh && prediction.mesh.length > 0) {
-      for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => mesh[index]);
+      for (const key of Object.keys(coords.MESH_ANNOTATIONS)) annotations[key] = coords.MESH_ANNOTATIONS[key].map((index) => prediction.mesh[index]);
     }
-    const box = prediction.box ? [
+    const clampedBox = prediction.box ? [
       Math.max(0, prediction.box.startPoint[0]),
       Math.max(0, prediction.box.startPoint[1]),
       Math.min(input.shape[2], prediction.box.endPoint[0]) - Math.max(0, prediction.box.startPoint[0]),
@@ -38,15 +37,14 @@ export async function predict(input, config): Promise<{ confidence, boxConfidenc
       confidence: Math.round(100 * prediction.faceConfidence || 100 * prediction.boxConfidence || 0) / 100,
       boxConfidence: Math.round(100 * prediction.boxConfidence) / 100,
       faceConfidence: Math.round(100 * prediction.faceConfidence) / 100,
-      box,
+      box: clampedBox,
       boxRaw,
-      mesh,
+      mesh: prediction.mesh,
       meshRaw,
       annotations,
-      image: prediction.image ? prediction.image.clone() : null,
+      image: prediction.image,
     });
     if (prediction.coords) prediction.coords.dispose();
-    if (prediction.image) prediction.image.dispose();
   }
   return results;
 }

src/blazeface/facepipeline.ts

@@ -90,9 +90,9 @@ export class Pipeline {
     const inverseRotationMatrix = (angle !== 0) ? util.invertTransformMatrix(rotationMatrix) : util.IDENTITY_MATRIX;
     const boxCenter = [...bounding.getBoxCenter({ startPoint: box.startPoint, endPoint: box.endPoint }), 1];
     return coordsRotated.map((coord) => ([
-      coord[0] + util.dot(boxCenter, inverseRotationMatrix[0]),
+      Math.round(coord[0] + util.dot(boxCenter, inverseRotationMatrix[0])),
-      coord[1] + util.dot(boxCenter, inverseRotationMatrix[1]),
+      Math.round(coord[1] + util.dot(boxCenter, inverseRotationMatrix[1])),
-      coord[2],
+      Math.round(coord[2]),
     ]));
   }
 
@@ -195,10 +195,7 @@ export class Pipeline {
         prediction.landmarks.dispose();
       });
     }
-
+    const results = tf.tidy(() => this.storedBoxes.map((box, i) => {
-    let results = tf.tidy(() => this.storedBoxes.map((box, i) => {
-      const boxConfidence = box.confidence;
-
       // The facial bounding box landmarks could come either from blazeface (if we are using a fresh box), or from the mesh model (if we are reusing an old box).
       let face;
       let angle = 0;
@@ -223,19 +220,22 @@ export class Pipeline {
       // if we're not going to produce mesh, don't spend time with further processing
       if (!config.face.mesh.enabled) {
         const prediction = {
-          coords: null,
+          mesh: [],
           box,
           faceConfidence: null,
-          boxConfidence,
+          boxConfidence: box.confidence,
           confidence: box.confidence,
           image: face,
         };
         return prediction;
       }
 
-      const [, confidence, contourCoords] = this.meshDetector.predict(face); // The first returned tensor represents facial contours which are already included in the coordinates.
+      const [, confidence, contourCoords] = this.meshDetector.execute(face); // The first returned tensor represents facial contours which are already included in the coordinates.
       const faceConfidence = confidence.dataSync()[0];
-      if (faceConfidence < config.face.detector.minConfidence) return null; // if below confidence just exit
+      if (faceConfidence < config.face.detector.minConfidence) {
+        this.storedBoxes[i].confidence = faceConfidence; // reset confidence of cached box
+        return null; // if below confidence just exit
+      }
       const coordsReshaped = tf.reshape(contourCoords, [-1, 3]);
       let rawCoords = coordsReshaped.arraySync();
 
@@ -265,9 +265,10 @@ export class Pipeline {
       }
 
       // override box from detection with one calculated from mesh
-      const transformedCoordsData = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
+      const mesh = this.transformRawCoords(rawCoords, box, angle, rotationMatrix);
-      box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(transformedCoordsData), 1.5); // redefine box with mesh calculated one
+      const storeConfidence = box.confidence;
-      const transformedCoords = tf.tensor2d(transformedCoordsData);
+      box = bounding.enlargeBox(bounding.calculateLandmarksBoundingBox(mesh), 1.5); // redefine box with mesh calculated one
+      box.confidence = storeConfidence;
 
       // do rotation one more time with mesh keypoints if we want to return perfect image
       if (config.face.detector.rotation && config.face.mesh.enabled && config.face.description.enabled && tf.ENV.flags.IS_BROWSER) {
@@ -281,24 +282,28 @@ export class Pipeline {
       }
 
       const prediction = {
-        coords: transformedCoords,
+        mesh,
         box,
         faceConfidence,
-        boxConfidence,
+        boxConfidence: box.confidence,
         image: face,
-        rawCoords,
       };
 
       // updated stored cache values
-      const squarifiedLandmarksBox = bounding.squarifyBox(box);
+      const storedBox = bounding.squarifyBox(box);
-      this.storedBoxes[i] = { ...squarifiedLandmarksBox, landmarks: transformedCoordsData, confidence: box.confidence, faceConfidence };
+      // @ts-ignore box itself doesn't have those properties, but we stored them for future use
+      storedBox.confidence = box.confidence;
+      // @ts-ignore box itself doesn't have those properties, but we stored them for future use
+      storedBox.faceConfidence = faceConfidence;
+      // this.storedBoxes[i] = { ...squarifiedLandmarksBox, confidence: box.confidence, faceConfidence };
+      this.storedBoxes[i] = storedBox;
 
       return prediction;
     }));
 
-    results = results.filter((a) => a !== null);
+    // results = results.filter((a) => a !== null);
     // remove cache entries for detected boxes on low confidence
-    if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.faceConfidence > config.face.detector.minConfidence);
+    if (config.face.mesh.enabled) this.storedBoxes = this.storedBoxes.filter((a) => a.confidence > config.face.detector.minConfidence);
     this.detectedFaces = results.length;
 
     return results;