ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch to async data reads

pull/193/head
Vladimir Mandic 2021-08-12 09:31:16 -04:00
parent 86abe96b81
commit 85ac90c4e9
26 changed files with 127 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -1,6 +1,6 @@
# @vladmandic/human
Version: **2.1.2**
Version: **2.1.3**
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
Author: **Vladimir Mandic <mandic00@live.com>**
@ -9,7 +9,10 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/08/09 mandic00@live.com
### **2.1.3** 2021/08/12 mandic00@live.com
### **origin/main** 2021/08/11 mandic00@live.com
- minor update
- replace movenet with lightning-v4

41
TODO.md
View File

@ -21,31 +21,50 @@ WebGL shader optimizations for faster load and initial detection
- Optical Flow: <https://docs.opencv.org/3.3.1/db/d7f/tutorial_js_lucas_kanade.html>
- TFLite Models: <https://js.tensorflow.org/api_tflite/0.0.1-alpha.4/>
<br>
## Known Issues
### Object Detection
Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS
*Target: `Human` v2.2 with `TFJS` v3.9*
- CenterNet with WASM: <https://github.com/tensorflow/tfjs/issues/5110>
- NanoDet with WASM: <https://github.com/tensorflow/tfjs/issues/4824>
<br>
### Face Detection
Enhanced rotation correction for face detection is not working in NodeJS due to missing kernel op in TFJS
Feature is automatically disabled in NodeJS without user impact
*Target: `Human` v2.2 with `TFJS` v3.9*
- BlazeFace rotation correction in NodeJS: <https://github.com/tensorflow/tfjs/issues/4066>
- Backend NodeJS missing kernel op `FlipLeftRight`
<https://github.com/tensorflow/tfjs/issues/4066>
*Target: `Human` v2.2 with `TFJS` v3.9*
- Backend NodeJS missing kernel op `RotateWithOffset`
<https://github.com/tensorflow/tfjs/issues/5473>
*Target: N/A*
<br>
### Hand Detection
Enhanced rotation correction for hand detection is not working in NodeJS due to missing kernel op in TFJS
Feature is automatically disabled in NodeJS without user impact
*Target: `Human` v2.2 with `TFJS` v3.9*
- HandPose rotation correction in NodeJS: <https://github.com/tensorflow/tfjs/issues/4066>
- Backend NodeJS missing kernel op `FlipLeftRight`
<https://github.com/tensorflow/tfjs/issues/4066>
*Target: `Human` v2.2 with `TFJS` v3.9*
- Backend NodeJS missing kernel op `RotateWithOffset`
<https://github.com/tensorflow/tfjs/issues/5473>
*Target: N/A*
Hand detection using WASM backend has reduced precision due to math rounding errors in backend
*Target: N/A*
<br>
### Object Detection
Object detection using CenterNet or NanoDet models is not working when using WASM backend due to missing kernel ops in TFJS
- Backend WASM missing kernel op `Mod`
<https://github.com/tensorflow/tfjs/issues/5110>
*Target: `Human` v2.2 with `TFJS` v3.9*
- Backend WASM missing kernel op `SparseToDense`
<https://github.com/tensorflow/tfjs/issues/4824>
*Target: `Human` v2.2 with `TFJS` v3.9*

2
demo/nodejs/node.js
View File

@ -121,7 +121,7 @@ async function detect(input) {
if (result && result.hand && result.hand.length > 0) {
for (let i = 0; i < result.hand.length; i++) {
const hand = result.hand[i];
log.data(` Hand: #${i} score:${hand.score}`);
log.data(` Hand: #${i} score:${hand.score} keypoints:${hand.keypoints?.length}`);
}
} else {
log.data(' Hand: N/A');

12
dist/human.esm-nobundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm-nobundle.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.esm.js vendored
View File

File diff suppressed because one or more lines are too long

4
dist/human.esm.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
dist/human.js vendored
View File

File diff suppressed because one or more lines are too long

20
dist/human.node-gpu.js vendored
View File

@ -4306,7 +4306,7 @@ async function predict3(image18, config3, idx, count2) {
const obj = [];
if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize);
const data = emotionT.dataSync();
const data = await emotionT.data();
tf7.dispose(emotionT);
for (let i = 0; i < data.length; i++) {
if (data[i] > config3.face.emotion.minConfidence)
@ -7704,7 +7704,7 @@ var HandDetector = class {
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync();
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
@ -7736,7 +7736,7 @@ var HandDetector = class {
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = prediction.box.dataSync();
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
@ -8125,7 +8125,6 @@ async function load7(config3) {
return model4;
}
async function predict6(image18, config3) {
var _a;
if (!model4)
return [];
if (!config3.body.enabled)
@ -8135,7 +8134,8 @@ async function predict6(image18, config3) {
const normalize = tf13.div(resize, [255]);
tf13.dispose(resize);
const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
const findT = resT.find((t) => t.size === 195 || t.size === 155);
const points = await (findT == null ? void 0 : findT.data()) || [];
resT.forEach((t) => tf13.dispose(t));
tf13.dispose(normalize);
const keypoints3 = [];
@ -8516,7 +8516,7 @@ async function process2(res, inputSize, outputShape, config3) {
let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf16.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
nmsIdx = nms.dataSync();
nmsIdx = await nms.data();
tf16.dispose(nms);
}
results = results.filter((_val, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
@ -8583,7 +8583,7 @@ async function process3(res, inputSize, outputShape, config3) {
tf17.dispose(boxesT);
tf17.dispose(scoresT);
tf17.dispose(classesT);
const nms = nmsT.dataSync();
const nms = await nmsT.data();
tf17.dispose(nmsT);
let i = 0;
for (const id of nms) {
@ -9527,7 +9527,7 @@ async function predict11(input) {
resizeOutput = tf19.image.resizeBilinear(squeeze7, [width, height]);
}
if (typeof document === "undefined")
return resizeOutput.dataSync();
return resizeOutput.data();
const overlay = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(width, height) : document.createElement("canvas");
overlay.width = width;
overlay.height = height;
@ -11331,7 +11331,7 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`;
// package.json
var version = "2.1.2";
var version = "2.1.3";
// src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _lastCacheDiff, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
@ -11427,7 +11427,7 @@ var Human = class {
return false;
const resizeFact = 32;
const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync();
const reducedData = await reduced.data();
let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++)
sum += reducedData[3 * i + 2];

20
dist/human.node-wasm.js vendored
View File

@ -4307,7 +4307,7 @@ async function predict3(image18, config3, idx, count2) {
const obj = [];
if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize);
const data = emotionT.dataSync();
const data = await emotionT.data();
tf7.dispose(emotionT);
for (let i = 0; i < data.length; i++) {
if (data[i] > config3.face.emotion.minConfidence)
@ -7705,7 +7705,7 @@ var HandDetector = class {
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync();
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
@ -7737,7 +7737,7 @@ var HandDetector = class {
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = prediction.box.dataSync();
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
@ -8126,7 +8126,6 @@ async function load7(config3) {
return model4;
}
async function predict6(image18, config3) {
var _a;
if (!model4)
return [];
if (!config3.body.enabled)
@ -8136,7 +8135,8 @@ async function predict6(image18, config3) {
const normalize = tf13.div(resize, [255]);
tf13.dispose(resize);
const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
const findT = resT.find((t) => t.size === 195 || t.size === 155);
const points = await (findT == null ? void 0 : findT.data()) || [];
resT.forEach((t) => tf13.dispose(t));
tf13.dispose(normalize);
const keypoints3 = [];
@ -8517,7 +8517,7 @@ async function process2(res, inputSize, outputShape, config3) {
let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf16.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
nmsIdx = nms.dataSync();
nmsIdx = await nms.data();
tf16.dispose(nms);
}
results = results.filter((_val, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
@ -8584,7 +8584,7 @@ async function process3(res, inputSize, outputShape, config3) {
tf17.dispose(boxesT);
tf17.dispose(scoresT);
tf17.dispose(classesT);
const nms = nmsT.dataSync();
const nms = await nmsT.data();
tf17.dispose(nmsT);
let i = 0;
for (const id of nms) {
@ -9528,7 +9528,7 @@ async function predict11(input) {
resizeOutput = tf19.image.resizeBilinear(squeeze7, [width, height]);
}
if (typeof document === "undefined")
return resizeOutput.dataSync();
return resizeOutput.data();
const overlay = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(width, height) : document.createElement("canvas");
overlay.width = width;
overlay.height = height;
@ -11332,7 +11332,7 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`;
// package.json
var version = "2.1.2";
var version = "2.1.3";
// src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _lastCacheDiff, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
@ -11428,7 +11428,7 @@ var Human = class {
return false;
const resizeFact = 32;
const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync();
const reducedData = await reduced.data();
let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++)
sum += reducedData[3 * i + 2];

20
dist/human.node.js vendored
View File

@ -4306,7 +4306,7 @@ async function predict3(image18, config3, idx, count2) {
const obj = [];
if (config3.face.emotion.enabled) {
const emotionT = await model2.predict(normalize);
const data = emotionT.dataSync();
const data = await emotionT.data();
tf7.dispose(emotionT);
for (let i = 0; i < data.length; i++) {
if (data[i] > config3.face.emotion.minConfidence)
@ -7704,7 +7704,7 @@ var HandDetector = class {
const predictions = tf10.squeeze(batched);
tf10.dispose(batched);
const scoresT = tf10.tidy(() => tf10.squeeze(tf10.sigmoid(tf10.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync();
const scores = await scoresT.data();
const rawBoxes = tf10.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf10.dispose(rawBoxes);
@ -7736,7 +7736,7 @@ var HandDetector = class {
if (!predictions || predictions.length === 0)
return hands;
for (const prediction of predictions) {
const boxes = prediction.box.dataSync();
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();
@ -8125,7 +8125,6 @@ async function load7(config3) {
return model4;
}
async function predict6(image18, config3) {
var _a;
if (!model4)
return [];
if (!config3.body.enabled)
@ -8135,7 +8134,8 @@ async function predict6(image18, config3) {
const normalize = tf13.div(resize, [255]);
tf13.dispose(resize);
const resT = await model4.predict(normalize);
const points = ((_a = resT.find((t) => t.size === 195 || t.size === 155)) == null ? void 0 : _a.dataSync()) || [];
const findT = resT.find((t) => t.size === 195 || t.size === 155);
const points = await (findT == null ? void 0 : findT.data()) || [];
resT.forEach((t) => tf13.dispose(t));
tf13.dispose(normalize);
const keypoints3 = [];
@ -8516,7 +8516,7 @@ async function process2(res, inputSize, outputShape, config3) {
let nmsIdx = [];
if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf16.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config3.object.maxDetected, config3.object.iouThreshold, config3.object.minConfidence);
nmsIdx = nms.dataSync();
nmsIdx = await nms.data();
tf16.dispose(nms);
}
results = results.filter((_val, idx) => nmsIdx.includes(idx)).sort((a, b) => b.score - a.score);
@ -8583,7 +8583,7 @@ async function process3(res, inputSize, outputShape, config3) {
tf17.dispose(boxesT);
tf17.dispose(scoresT);
tf17.dispose(classesT);
const nms = nmsT.dataSync();
const nms = await nmsT.data();
tf17.dispose(nmsT);
let i = 0;
for (const id of nms) {
@ -9527,7 +9527,7 @@ async function predict11(input) {
resizeOutput = tf19.image.resizeBilinear(squeeze7, [width, height]);
}
if (typeof document === "undefined")
return resizeOutput.dataSync();
return resizeOutput.data();
const overlay = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(width, height) : document.createElement("canvas");
overlay.width = width;
overlay.height = height;
@ -11331,7 +11331,7 @@ lBhEMohlFerLlBjEMohMVTEARDKCITsAk2AEgAAAkAAAAAAAAAAAAAAAAAAAAAAAASAAAAAAAAD/
2Q==`;
// package.json
var version = "2.1.2";
var version = "2.1.3";
// src/human.ts
var _numTensors, _analyzeMemoryLeaks, _checkSanity, _firstRun, _lastInputSum, _lastCacheDiff, _sanity, _checkBackend, _skipFrame, _warmupBitmap, _warmupCanvas, _warmupNode;
@ -11427,7 +11427,7 @@ var Human = class {
return false;
const resizeFact = 32;
const reduced = tf21.image.resizeBilinear(input, [Math.trunc(input.shape[1] / resizeFact), Math.trunc(input.shape[2] / resizeFact)]);
const reducedData = reduced.dataSync();
const reducedData = await reduced.data();
let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++)
sum += reducedData[3 * i + 2];

44
server/build.log
View File

@ -1,22 +1,22 @@
2021-08-11 18:57:35 INFO:  @vladmandic/human version 2.1.2
2021-08-11 18:57:35 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-11 18:57:35 INFO:  Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.19","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"}
2021-08-11 18:57:35 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-11 18:57:35 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-11 18:57:35 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-11 18:57:35 STATE: target: node type: node: {"imports":42,"importBytes":436067,"outputBytes":377952,"outputFiles":"dist/human.node.js"}
2021-08-11 18:57:35 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-11 18:57:35 STATE: target: nodeGPU type: node: {"imports":42,"importBytes":436075,"outputBytes":377956,"outputFiles":"dist/human.node-gpu.js"}
2021-08-11 18:57:35 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-11 18:57:35 STATE: target: nodeWASM type: node: {"imports":42,"importBytes":436142,"outputBytes":378028,"outputFiles":"dist/human.node-wasm.js"}
2021-08-11 18:57:35 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-11 18:57:35 STATE: target: browserNoBundle type: esm: {"imports":42,"importBytes":436006,"outputBytes":248005,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-11 18:57:36 STATE: target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
2021-08-11 18:57:36 STATE: target: browserBundle type: iife: {"imports":42,"importBytes":2769465,"outputBytes":1378447,"outputFiles":"dist/human.js"}
2021-08-11 18:57:36 STATE: target: browserBundle type: esm: {"imports":42,"importBytes":2769465,"outputBytes":1378439,"outputFiles":"dist/human.esm.js"}
2021-08-11 18:57:36 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-11 18:57:59 INFO:  Linter complete: files: 75 errors: 0 warnings: 0
2021-08-11 18:57:59 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-11 18:57:59 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-11 18:58:13 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-11 18:58:27 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1
2021-08-12 09:29:51 INFO:  @vladmandic/human version 2.1.3
2021-08-12 09:29:51 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.5.0
2021-08-12 09:29:51 INFO:  Toolchain: {"tfjs":"3.8.0","esbuild":"0.12.19","typescript":"4.3.5","typedoc":"0.21.5","eslint":"7.32.0"}
2021-08-12 09:29:51 INFO:  Clean: ["dist/*","types/*","typedoc/*"]
2021-08-12 09:29:51 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true}
2021-08-12 09:29:51 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"}
2021-08-12 09:29:51 STATE: target: node type: node: {"imports":42,"importBytes":436279,"outputBytes":377971,"outputFiles":"dist/human.node.js"}
2021-08-12 09:29:51 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"}
2021-08-12 09:29:51 STATE: target: nodeGPU type: node: {"imports":42,"importBytes":436287,"outputBytes":377975,"outputFiles":"dist/human.node-gpu.js"}
2021-08-12 09:29:51 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"}
2021-08-12 09:29:51 STATE: target: nodeWASM type: node: {"imports":42,"importBytes":436354,"outputBytes":378047,"outputFiles":"dist/human.node-wasm.js"}
2021-08-12 09:29:51 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2168,"outputBytes":1242,"outputFiles":"dist/tfjs.esm.js"}
2021-08-12 09:29:51 STATE: target: browserNoBundle type: esm: {"imports":42,"importBytes":436218,"outputBytes":248008,"outputFiles":"dist/human.esm-nobundle.js"}
2021-08-12 09:29:51 STATE: target: browserBundle type: tfjs: {"modules":1170,"moduleBytes":4145868,"imports":7,"importBytes":2168,"outputBytes":2334701,"outputFiles":"dist/tfjs.esm.js"}
2021-08-12 09:29:52 STATE: target: browserBundle type: iife: {"imports":42,"importBytes":2769677,"outputBytes":1378450,"outputFiles":"dist/human.js"}
2021-08-12 09:29:52 STATE: target: browserBundle type: esm: {"imports":42,"importBytes":2769677,"outputBytes":1378442,"outputFiles":"dist/human.esm.js"}
2021-08-12 09:29:52 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"]
2021-08-12 09:30:14 INFO:  Linter complete: files: 75 errors: 0 warnings: 0
2021-08-12 09:30:14 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"]
2021-08-12 09:30:14 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"]
2021-08-12 09:30:29 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"]
2021-08-12 09:30:43 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1

2
src/age/age.ts
View File

@ -45,7 +45,7 @@ export async function predict(image: Tensor, config: Config | any) {
tf.dispose(enhance);
if (ageT) {
const data = ageT.dataSync();
const data = await ageT.data();
obj.age = Math.trunc(10 * data[0]) / 10;
}
tf.dispose(ageT);

2
src/blazeface/blazeface.ts
View File

@ -57,7 +57,7 @@ export class BlazeFaceModel {
}
const boxesOut = decodeBounds(batchOut, this.anchors, [this.inputSize, this.inputSize]);
const logits = tf.slice(batchOut, [0, 0], [-1, 1]);
const scoresOut = tf.squeeze(tf.sigmoid(logits)).dataSync();
const scoresOut = tf.squeeze(tf.sigmoid(logits)).dataSync(); // inside tf.tidy
return [batchOut, boxesOut, scoresOut];
});

4
src/blazeface/facepipeline.ts
View File

@ -234,7 +234,7 @@ export class Pipeline {
}
const [, confidence, contourCoords] = this.meshDetector.execute(face) as Array<Tensor>; // The first returned tensor represents facial contours which are already included in the coordinates.
const faceConfidence = confidence.dataSync()[0] as number;
const faceConfidence = confidence.dataSync()[0] as number; // inside tf.tidy
if (faceConfidence < config.face.detector.minConfidence) {
this.storedBoxes[i].confidence = faceConfidence; // reset confidence of cached box
return null; // if below confidence just exit
@ -246,7 +246,7 @@ export class Pipeline {
const { box: leftEyeBox, boxSize: leftEyeBoxSize, crop: leftEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.leftBounds[0], eyeLandmarks.leftBounds[1], true);
const { box: rightEyeBox, boxSize: rightEyeBoxSize, crop: rightEyeCrop } = this.getEyeBox(rawCoords, face, eyeLandmarks.rightBounds[0], eyeLandmarks.rightBounds[1]);
const eyePredictions = this.irisModel.predict(tf.concat([leftEyeCrop, rightEyeCrop])) as Tensor;
const eyePredictionsData = eyePredictions.dataSync();
const eyePredictionsData = eyePredictions.dataSync(); // inside tf.tidy
const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3);
const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = this.getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true);
const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3);

3
src/blazepose/blazepose.ts
View File

@ -33,7 +33,8 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
const normalize = tf.div(resize, [255.0]);
tf.dispose(resize);
const resT = await model.predict(normalize) as Array<Tensor>;
const points = resT.find((t) => (t.size === 195 || t.size === 155))?.dataSync() || []; // order of output tensors may change between models, full has 195 and upper has 155 items
const findT = resT.find((t) => (t.size === 195 || t.size === 155));
const points = await findT?.data() || []; // order of output tensors may change between models, full has 195 and upper has 155 items
resT.forEach((t) => tf.dispose(t));
tf.dispose(normalize);
const keypoints: Array<{ id, part, position: [number, number, number], positionRaw: [number, number, number], score, presence }> = [];

6
src/efficientpose/efficientpose.ts
View File

@ -39,12 +39,12 @@ function max2d(inputs, minScore) {
// combine all data
const reshaped = tf.reshape(inputs, [height * width]);
// get highest score
const newScore = tf.max(reshaped, 0).dataSync()[0];
const newScore = tf.max(reshaped, 0).dataSync()[0]; // inside tf.tidy
if (newScore > minScore) {
// skip coordinate calculation is score is too low
const coords = tf.argMax(reshaped, 0);
const x = mod(coords, width).dataSync()[0];
const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0];
const x = mod(coords, width).dataSync()[0]; // inside tf.tidy
const y = tf.div(coords, tf.scalar(width, 'int32')).dataSync()[0]; // inside tf.tidy
return [x, y, newScore];
}
return [0, 0, newScore];

2
src/embedding/embedding.ts
View File

@ -125,7 +125,7 @@ export async function predict(input, config): Promise<number[]> {
const reshape = tf.reshape(res, [128, 2]); // split 256 vectors into 128 x 2
const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it
const output: Array<number> = reduce.dataSync();
const output: Array<number> = reduce.dataSync(); // inside tf.tidy
return [...output]; // convert typed array to simple array
});
tf.dispose(image);

2
src/emotion/emotion.ts
View File

@ -53,7 +53,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion.enabled) {
const emotionT = await model.predict(normalize); // result is already in range 0..1, no need for additional activation
const data = emotionT.dataSync();
const data = await emotionT.data();
tf.dispose(emotionT);
for (let i = 0; i < data.length; i++) {
if (data[i] > config.face.emotion.minConfidence) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });

8
src/faceres/faceres.ts
View File

@ -134,21 +134,21 @@ export async function predict(image: Tensor, config: Config, idx, count) {
if (resT) {
tf.tidy(() => {
const gender = resT.find((t) => t.shape[1] === 1).dataSync();
const gender = resT.find((t) => t.shape[1] === 1).dataSync(); // inside tf.tidy
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
if (confidence > config.face.description.minConfidence) {
obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
obj.genderScore = Math.min(0.99, confidence);
}
const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0];
const all = resT.find((t) => t.shape[1] === 100).dataSync();
const age = tf.argMax(resT.find((t) => t.shape[1] === 100), 1).dataSync()[0]; // inside tf.tidy
const all = resT.find((t) => t.shape[1] === 100).dataSync(); // inside tf.tidy
obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024);
// const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
// const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
obj.descriptor = [...desc.dataSync()];
obj.descriptor = [...desc.dataSync()]; // inside tf.tidy
});
resT.forEach((t) => tf.dispose(t));
}

4
src/gender/gender.ts
View File

@ -63,7 +63,7 @@ export async function predict(image: Tensor, config: Config | any) {
if (genderT) {
if (!Array.isArray(genderT)) {
const data = genderT.dataSync();
const data = await genderT.data();
if (alternative) {
// returns two values 0..1, bigger one is prediction
if (data[0] > config.face.gender.minConfidence || data[1] > config.face.gender.minConfidence) {
@ -80,7 +80,7 @@ export async function predict(image: Tensor, config: Config | any) {
}
tf.dispose(genderT);
} else {
const gender = genderT[0].dataSync();
const gender = await genderT[0].data();
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
if (confidence > config.face.gender.minConfidence) {
obj.gender = gender[0] <= 0.5 ? 'female' : 'male';

4
src/handpose/handdetector.ts
View File

@ -45,7 +45,7 @@ export class HandDetector {
const predictions = tf.squeeze(batched);
tf.dispose(batched);
const scoresT = tf.tidy(() => tf.squeeze(tf.sigmoid(tf.slice(predictions, [0, 0], [-1, 1]))));
const scores = scoresT.dataSync();
const scores = await scoresT.data();
const rawBoxes = tf.slice(predictions, [0, 1], [-1, 4]);
const boxes = this.normalizeBoxes(rawBoxes);
tf.dispose(rawBoxes);
@ -78,7 +78,7 @@ export class HandDetector {
const hands: Array<{ startPoint: number[]; endPoint: number[]; palmLandmarks: number[]; confidence: number }> = [];
if (!predictions || predictions.length === 0) return hands;
for (const prediction of predictions) {
const boxes = prediction.box.dataSync();
const boxes = await prediction.box.data();
const startPoint = boxes.slice(0, 2);
const endPoint = boxes.slice(2, 4);
const palmLandmarks = await prediction.palmLandmarks.array();

2
src/human.ts
View File

@ -367,7 +367,7 @@ export class Human {
sumT.dispose();
*/
// use js loop sum, faster than uploading tensor to gpu calculating and downloading back
const reducedData = reduced.dataSync(); // raw image rgb array
const reducedData = await reduced.data(); // raw image rgb array
let sum = 0;
for (let i = 0; i < reducedData.length / 3; i++) sum += reducedData[3 * i + 2]; // look only at green value of each pixel

2
src/object/centernet.ts
View File

@ -42,7 +42,7 @@ async function process(res: Tensor, inputSize, outputShape, config: Config) {
tf.dispose(boxesT);
tf.dispose(scoresT);
tf.dispose(classesT);
const nms = nmsT.dataSync();
const nms = await nmsT.data();
tf.dispose(nmsT);
let i = 0;
for (const id of nms) {

2
src/object/nanodet.ts
View File

@ -90,7 +90,7 @@ async function process(res, inputSize, outputShape, config) {
let nmsIdx: Array<number> = [];
if (nmsBoxes && nmsBoxes.length > 0) {
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
nmsIdx = nms.dataSync();
nmsIdx = await nms.data();
tf.dispose(nms);
}

2
src/segmentation/segmentation.ts
View File

@ -59,7 +59,7 @@ export async function predict(input: { tensor: Tensor | null, canvas: OffscreenC
resizeOutput = tf.image.resizeBilinear(squeeze, [width, height]);
}
if (typeof document === 'undefined') return resizeOutput.dataSync(); // we're running in nodejs so return alpha array as-is
if (typeof document === 'undefined') return resizeOutput.data(); // we're running in nodejs so return alpha array as-is
const overlay = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(width, height) : document.createElement('canvas');
overlay.width = width;