mirror of https://github.com/vladmandic/human
fix centernet & update blazeface
Vladimir Mandic
parent
2eae119c96
commit
67b7db377d
|
|
@ -9,7 +9,7 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2021/08/06 mandic00@live.com
|
||||
### **HEAD -> main** 2021/08/09 mandic00@live.com
|
||||
|
||||
- minor update
|
||||
- replace movenet with lightning-v4
|
||||
|
|
|
|||
|
|
@ -40,15 +40,15 @@ let userConfig = {
|
|||
enabled: false,
|
||||
flip: false,
|
||||
},
|
||||
face: { enabled: false,
|
||||
detector: { return: true },
|
||||
face: { enabled: true,
|
||||
detector: { return: false },
|
||||
mesh: { enabled: true },
|
||||
iris: { enabled: false },
|
||||
description: { enabled: false },
|
||||
emotion: { enabled: false },
|
||||
},
|
||||
object: { enabled: false },
|
||||
gesture: { enabled: true },
|
||||
gesture: { enabled: false },
|
||||
hand: { enabled: false },
|
||||
body: { enabled: false },
|
||||
// body: { enabled: true, modelPath: 'posenet.json' },
|
||||
|
|
@ -164,7 +164,8 @@ function status(msg) {
|
|||
div.innerText = msg;
|
||||
} else {
|
||||
const video = document.getElementById('video');
|
||||
document.getElementById('play').style.display = (video.srcObject !== null) && !video.paused ? 'none' : 'block';
|
||||
const playing = (video.srcObject !== null) && !video.paused;
|
||||
document.getElementById('play').style.display = playing ? 'none' : 'block';
|
||||
document.getElementById('loader').style.display = 'none';
|
||||
div.innerText = '';
|
||||
}
|
||||
|
|
@ -259,9 +260,10 @@ async function drawResults(input) {
|
|||
const avgDraw = ui.drawFPS.length > 0 ? Math.trunc(10 * ui.drawFPS.reduce((a, b) => a + b, 0) / ui.drawFPS.length) / 10 : 0;
|
||||
const warning = (ui.detectFPS.length > 5) && (avgDetect < 2) ? '<font color="lightcoral">warning: your performance is low: try switching to higher performance backend, lowering resolution or disabling some models</font>' : '';
|
||||
const fps = avgDetect > 0 ? `FPS process:${avgDetect} refresh:${avgDraw}` : '';
|
||||
const backend = engine.state.numTensors > 0 ? `backend: ${human.tf.getBackend()} | ${memory}` : 'running in web worker';
|
||||
document.getElementById('log').innerHTML = `
|
||||
video: ${ui.camera.name} | facing: ${ui.camera.facing} | screen: ${window.innerWidth} x ${window.innerHeight} camera: ${ui.camera.width} x ${ui.camera.height} ${processing}<br>
|
||||
backend: ${human.tf.getBackend()} | ${memory}<br>
|
||||
backend: ${backend}<br>
|
||||
performance: ${str(lastDetectedResult.performance)}ms ${fps}<br>
|
||||
${warning}<br>
|
||||
`;
|
||||
|
|
@ -363,6 +365,7 @@ async function setupCamera() {
|
|||
// eslint-disable-next-line no-use-before-define
|
||||
if (live && !ui.detectThread) runHumanDetect(video, canvas);
|
||||
ui.busy = false;
|
||||
status();
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
|
|
@ -597,6 +600,7 @@ async function detectVideo() {
|
|||
document.getElementById('btnStartText').innerHTML = 'pause video';
|
||||
await video.play();
|
||||
runHumanDetect(video, canvas);
|
||||
status();
|
||||
} else {
|
||||
status(cameraError);
|
||||
}
|
||||
|
|
@ -878,6 +882,7 @@ async function pwaRegister() {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
/*
|
||||
window.addEventListener('unhandledrejection', (evt) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(evt.reason || evt);
|
||||
|
|
@ -885,6 +890,7 @@ async function main() {
|
|||
status('exception error');
|
||||
evt.preventDefault();
|
||||
});
|
||||
*/
|
||||
|
||||
log('demo starting ...');
|
||||
|
||||
|
|
|
|||
|
|
@ -59,15 +59,16 @@
|
|||
"@tensorflow/tfjs-backend-cpu": "^3.8.0",
|
||||
"@tensorflow/tfjs-backend-wasm": "^3.8.0",
|
||||
"@tensorflow/tfjs-backend-webgl": "^3.8.0",
|
||||
"@tensorflow/tfjs-backend-webgpu": "^0.0.1-alpha.7",
|
||||
"@tensorflow/tfjs-converter": "^3.8.0",
|
||||
"@tensorflow/tfjs-core": "^3.8.0",
|
||||
"@tensorflow/tfjs-data": "^3.8.0",
|
||||
"@tensorflow/tfjs-layers": "^3.8.0",
|
||||
"@tensorflow/tfjs-node": "^3.8.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.8.0",
|
||||
"@types/node": "^16.4.13",
|
||||
"@typescript-eslint/eslint-plugin": "^4.29.0",
|
||||
"@typescript-eslint/parser": "^4.29.0",
|
||||
"@types/node": "^16.4.14",
|
||||
"@typescript-eslint/eslint-plugin": "^4.29.1",
|
||||
"@typescript-eslint/parser": "^4.29.1",
|
||||
"@vladmandic/pilogger": "^0.2.18",
|
||||
"canvas": "^2.8.0",
|
||||
"chokidar": "^3.5.2",
|
||||
|
|
|
|||
28
src/face.ts
28
src/face.ts
|
|
@ -163,8 +163,8 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
|
||||
// is something went wrong, skip the face
|
||||
// @ts-ignore possibly undefined
|
||||
if (!faces[i].image || faces[i].image['isDisposedInternal']) {
|
||||
log('Face object is disposed:', faces[i].image);
|
||||
if (!faces[i].tensor || faces[i].tensor['isDisposedInternal']) {
|
||||
log('Face object is disposed:', faces[i].tensor);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -173,11 +173,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
// run emotion, inherits face from blazeface
|
||||
parent.analyze('Start Emotion:');
|
||||
if (parent.config.async) {
|
||||
emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
} else {
|
||||
parent.state = 'run:emotion';
|
||||
timeStamp = now();
|
||||
emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
parent.performance.emotion = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
parent.analyze('End Emotion:');
|
||||
|
|
@ -186,11 +186,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
/*
|
||||
parent.analyze('Start GEAR:');
|
||||
if (parent.config.async) {
|
||||
gearRes = parent.config.face.agegenderrace.enabled ? agegenderrace.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
gearRes = parent.config.face.agegenderrace.enabled ? agegenderrace.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
} else {
|
||||
parent.state = 'run:gear';
|
||||
timeStamp = now();
|
||||
gearRes = parent.config.face.agegenderrace.enabled ? await agegenderrace.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
gearRes = parent.config.face.agegenderrace.enabled ? await agegenderrace.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {};
|
||||
parent.performance.emotion = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
parent.analyze('End GEAR:');
|
||||
|
|
@ -199,11 +199,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
// run emotion, inherits face from blazeface
|
||||
parent.analyze('Start Description:');
|
||||
if (parent.config.async) {
|
||||
descRes = parent.config.face.description.enabled ? faceres.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : [];
|
||||
descRes = parent.config.face.description.enabled ? faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : [];
|
||||
} else {
|
||||
parent.state = 'run:description';
|
||||
timeStamp = now();
|
||||
descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : [];
|
||||
descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : [];
|
||||
parent.performance.embedding = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
parent.analyze('End Description:');
|
||||
|
|
@ -226,6 +226,12 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
? Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) / input.shape[2]
|
||||
: 0;
|
||||
|
||||
// optionally return tensor
|
||||
const tensor = parent.config.face.detector.return ? tf.squeeze(faces[i].tensor) : null;
|
||||
// dispose original face tensor
|
||||
tf.dispose(faces[i].tensor);
|
||||
// delete temp face image
|
||||
if (faces[i].tensor) delete faces[i].tensor;
|
||||
// combine results
|
||||
faceRes.push({
|
||||
...faces[i],
|
||||
|
|
@ -237,12 +243,8 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
|
|||
emotion: emotionRes,
|
||||
iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0,
|
||||
rotation,
|
||||
tensor: parent.config.face.detector.return ? tf.squeeze(faces[i].image) : null,
|
||||
tensor,
|
||||
});
|
||||
// dispose original face tensor
|
||||
tf.dispose(faces[i].image);
|
||||
// delete temp face image
|
||||
if (faces[i].image) delete faces[i].image;
|
||||
|
||||
parent.analyze('End Face');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ export class HandDetector {
|
|||
const boxes = this.normalizeBoxes(rawBoxes);
|
||||
tf.dispose(rawBoxes);
|
||||
const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence);
|
||||
const filtered = filteredT.arraySync();
|
||||
const filtered = await filteredT.array();
|
||||
|
||||
tf.dispose(scoresT);
|
||||
tf.dispose(filteredT);
|
||||
|
|
@ -81,7 +81,7 @@ export class HandDetector {
|
|||
const boxes = prediction.box.dataSync();
|
||||
const startPoint = boxes.slice(0, 2);
|
||||
const endPoint = boxes.slice(2, 4);
|
||||
const palmLandmarks = prediction.palmLandmarks.arraySync();
|
||||
const palmLandmarks = await prediction.palmLandmarks.array();
|
||||
tf.dispose(prediction.box);
|
||||
tf.dispose(prediction.palmLandmarks);
|
||||
hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize]));
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ export class HandPipeline {
|
|||
tf.dispose(confidenceT);
|
||||
if (confidence >= config.hand.minConfidence) {
|
||||
const keypointsReshaped = tf.reshape(keypoints, [-1, 3]);
|
||||
const rawCoords = keypointsReshaped.arraySync();
|
||||
const rawCoords = await keypointsReshaped.array();
|
||||
tf.dispose(keypoints);
|
||||
tf.dispose(keypointsReshaped);
|
||||
const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix);
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ export async function predict(image: Tensor, config: Config): Promise<Body[]> {
|
|||
|
||||
if (resT) {
|
||||
keypoints.length = 0;
|
||||
const res = resT.arraySync();
|
||||
const res = await resT.array();
|
||||
tf.dispose(resT);
|
||||
const kpt = res[0][0];
|
||||
for (let id = 0; id < kpt.length; id++) {
|
||||
|
|
|
|||
|
|
@ -28,15 +28,15 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
async function process(res: Tensor, inputSize, outputShape, config: Config) {
|
||||
if (!res) return [];
|
||||
const results: Array<Item> = [];
|
||||
const detections = res.arraySync();
|
||||
const detections = await res.array();
|
||||
const squeezeT = tf.squeeze(res);
|
||||
tf.dispose(res);
|
||||
const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class
|
||||
tf.dispose(squeezeT);
|
||||
const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x
|
||||
const boxesT = stackT.squeeze();
|
||||
const scoresT = arr[4].squeeze();
|
||||
const classesT = arr[5].squeeze();
|
||||
const boxesT = tf.squeeze(stackT);
|
||||
const scoresT = tf.squeeze(arr[4]);
|
||||
const classesT = tf.squeeze(arr[5]);
|
||||
arr.forEach((t) => tf.dispose(t));
|
||||
const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
|
||||
tf.dispose(boxesT);
|
||||
|
|
|
|||
|
|
@ -32,14 +32,14 @@ async function process(res, inputSize, outputShape, config) {
|
|||
let results: Array<Item> = [];
|
||||
for (const strideSize of [1, 2, 4]) { // try each stride size as it detects large/medium/small objects
|
||||
// find scores, boxes, classes
|
||||
tf.tidy(() => { // wrap in tidy to automatically deallocate temp tensors
|
||||
tf.tidy(async () => { // wrap in tidy to automatically deallocate temp tensors
|
||||
const baseSize = strideSize * 13; // 13x13=169, 26x26=676, 52x52=2704
|
||||
// find boxes and scores output depending on stride
|
||||
const scoresT = res.find((a) => (a.shape[1] === (baseSize ** 2) && a.shape[2] === labels.length))?.squeeze();
|
||||
const featuresT = res.find((a) => (a.shape[1] === (baseSize ** 2) && a.shape[2] < labels.length))?.squeeze();
|
||||
const boxesMax = featuresT.reshape([-1, 4, featuresT.shape[1] / 4]); // reshape [output] to [4, output / 4] where number is number of different features inside each stride
|
||||
const boxIdx = boxesMax.argMax(2).arraySync(); // what we need is indexes of features with highest scores, not values itself
|
||||
const scores = scoresT.arraySync(); // optionally use exponential scores or just as-is
|
||||
const boxIdx = await boxesMax.argMax(2).array(); // what we need is indexes of features with highest scores, not values itself
|
||||
const scores = await scoresT.array(); // optionally use exponential scores or just as-is
|
||||
for (let i = 0; i < scoresT.shape[0]; i++) { // total strides (x * y matrix)
|
||||
for (let j = 0; j < scoresT.shape[1]; j++) { // one score for each class
|
||||
const score = scores[i][j]; // get score for current position
|
||||
|
|
|
|||
|
|
@ -53,8 +53,7 @@ export interface Face {
|
|||
matrix: [number, number, number, number, number, number, number, number, number],
|
||||
gaze: { bearing: number, strength: number },
|
||||
}
|
||||
image?: Tensor;
|
||||
tensor: Tensor,
|
||||
tensor?: Tensor,
|
||||
}
|
||||
|
||||
/** Body results
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
* @external
|
||||
*/
|
||||
|
||||
// import from src
|
||||
// get versions of all packages
|
||||
import { version as tfjsVersion } from '@tensorflow/tfjs/package.json';
|
||||
import { version as tfjsCoreVersion } from '@tensorflow/tfjs-core/package.json';
|
||||
|
|
@ -14,7 +13,7 @@ import { version as tfjsBackendCPUVersion } from '@tensorflow/tfjs-backend-cpu/p
|
|||
import { version as tfjsBackendWebGLVersion } from '@tensorflow/tfjs-backend-webgl/package.json';
|
||||
import { version as tfjsBackendWASMVersion } from '@tensorflow/tfjs-backend-wasm/package.json';
|
||||
|
||||
// export all
|
||||
// export all from sources
|
||||
// requires treeShaking:ignore-annotations due to tfjs misconfiguration
|
||||
/*
|
||||
export * from '@tensorflow/tfjs-core/src/index';
|
||||
|
|
@ -26,7 +25,7 @@ export * from '@tensorflow/tfjs-backend-webgl/src/index';
|
|||
export * from '@tensorflow/tfjs-backend-wasm/src/index';
|
||||
*/
|
||||
|
||||
// export all
|
||||
// export all from build
|
||||
export * from '@tensorflow/tfjs-core/dist/index.js';
|
||||
export * from '@tensorflow/tfjs-layers/dist/index.js';
|
||||
export * from '@tensorflow/tfjs-converter/dist/index.js';
|
||||
|
|
@ -34,6 +33,7 @@ export * as data from '@tensorflow/tfjs-data/dist/index.js';
|
|||
export * from '@tensorflow/tfjs-backend-cpu/dist/index.js';
|
||||
export * from '@tensorflow/tfjs-backend-webgl/dist/index.js';
|
||||
export * from '@tensorflow/tfjs-backend-wasm/dist/index.js';
|
||||
// export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js'; // experimental
|
||||
|
||||
// export versions
|
||||
export const version = {
|
||||
|
|
@ -46,4 +46,3 @@ export const version = {
|
|||
'tfjs-backend-webgl': tfjsBackendWebGLVersion,
|
||||
'tfjs-backend-wasm': tfjsBackendWASMVersion,
|
||||
};
|
||||
// export const version = {};
|
||||
|
|
|
|||
2
wiki
2
wiki
|
|
@ -1 +1 @@
|
|||
Subproject commit 2135debf198b5b0ecb670896bef837cbb45fe32e
|
||||
Subproject commit bdc4077a3df07abdf4a2d5b2d2beadf2e573e8d8
|
||||
Loading…
Reference in New Issue