diff --git a/CHANGELOG.md b/CHANGELOG.md index 86855d34..1c582f24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/08/06 mandic00@live.com +### **HEAD -> main** 2021/08/09 mandic00@live.com - minor update - replace movenet with lightning-v4 diff --git a/demo/index.js b/demo/index.js index e2e8316d..e0e3b465 100644 --- a/demo/index.js +++ b/demo/index.js @@ -40,15 +40,15 @@ let userConfig = { enabled: false, flip: false, }, - face: { enabled: false, - detector: { return: true }, + face: { enabled: true, + detector: { return: false }, mesh: { enabled: true }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false }, }, object: { enabled: false }, - gesture: { enabled: true }, + gesture: { enabled: false }, hand: { enabled: false }, body: { enabled: false }, // body: { enabled: true, modelPath: 'posenet.json' }, @@ -164,7 +164,8 @@ function status(msg) { div.innerText = msg; } else { const video = document.getElementById('video'); - document.getElementById('play').style.display = (video.srcObject !== null) && !video.paused ? 'none' : 'block'; + const playing = (video.srcObject !== null) && !video.paused; + document.getElementById('play').style.display = playing ? 'none' : 'block'; document.getElementById('loader').style.display = 'none'; div.innerText = ''; } @@ -259,9 +260,10 @@ async function drawResults(input) { const avgDraw = ui.drawFPS.length > 0 ? Math.trunc(10 * ui.drawFPS.reduce((a, b) => a + b, 0) / ui.drawFPS.length) / 10 : 0; const warning = (ui.detectFPS.length > 5) && (avgDetect < 2) ? 'warning: your performance is low: try switching to higher performance backend, lowering resolution or disabling some models' : ''; const fps = avgDetect > 0 ? `FPS process:${avgDetect} refresh:${avgDraw}` : ''; + const backend = engine.state.numTensors > 0 ? `backend: ${human.tf.getBackend()} | ${memory}` : 'running in web worker'; document.getElementById('log').innerHTML = ` video: ${ui.camera.name} | facing: ${ui.camera.facing} | screen: ${window.innerWidth} x ${window.innerHeight} camera: ${ui.camera.width} x ${ui.camera.height} ${processing}
- backend: ${human.tf.getBackend()} | ${memory}
+ backend: ${backend}
performance: ${str(lastDetectedResult.performance)}ms ${fps}
${warning}
`; @@ -363,6 +365,7 @@ async function setupCamera() { // eslint-disable-next-line no-use-before-define if (live && !ui.detectThread) runHumanDetect(video, canvas); ui.busy = false; + status(); resolve(); }; }); @@ -597,6 +600,7 @@ async function detectVideo() { document.getElementById('btnStartText').innerHTML = 'pause video'; await video.play(); runHumanDetect(video, canvas); + status(); } else { status(cameraError); } @@ -878,6 +882,7 @@ async function pwaRegister() { } async function main() { + /* window.addEventListener('unhandledrejection', (evt) => { // eslint-disable-next-line no-console console.error(evt.reason || evt); @@ -885,6 +890,7 @@ async function main() { status('exception error'); evt.preventDefault(); }); + */ log('demo starting ...'); diff --git a/package.json b/package.json index 1bfee12e..780ddae7 100644 --- a/package.json +++ b/package.json @@ -59,15 +59,16 @@ "@tensorflow/tfjs-backend-cpu": "^3.8.0", "@tensorflow/tfjs-backend-wasm": "^3.8.0", "@tensorflow/tfjs-backend-webgl": "^3.8.0", + "@tensorflow/tfjs-backend-webgpu": "^0.0.1-alpha.7", "@tensorflow/tfjs-converter": "^3.8.0", "@tensorflow/tfjs-core": "^3.8.0", "@tensorflow/tfjs-data": "^3.8.0", "@tensorflow/tfjs-layers": "^3.8.0", "@tensorflow/tfjs-node": "^3.8.0", "@tensorflow/tfjs-node-gpu": "^3.8.0", - "@types/node": "^16.4.13", - "@typescript-eslint/eslint-plugin": "^4.29.0", - "@typescript-eslint/parser": "^4.29.0", + "@types/node": "^16.4.14", + "@typescript-eslint/eslint-plugin": "^4.29.1", + "@typescript-eslint/parser": "^4.29.1", "@vladmandic/pilogger": "^0.2.18", "canvas": "^2.8.0", "chokidar": "^3.5.2", diff --git a/src/face.ts b/src/face.ts index 6473fa4f..dcf7fc08 100644 --- a/src/face.ts +++ b/src/face.ts @@ -163,8 +163,8 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): // is something went wrong, skip the face // @ts-ignore possibly undefined - if (!faces[i].image || faces[i].image['isDisposedInternal']) { - log('Face object is disposed:', faces[i].image); + if (!faces[i].tensor || faces[i].tensor['isDisposedInternal']) { + log('Face object is disposed:', faces[i].tensor); continue; } @@ -173,11 +173,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): // run emotion, inherits face from blazeface parent.analyze('Start Emotion:'); if (parent.config.async) { - emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {}; + emotionRes = parent.config.face.emotion.enabled ? emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {}; } else { parent.state = 'run:emotion'; timeStamp = now(); - emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {}; + emotionRes = parent.config.face.emotion.enabled ? await emotion.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {}; parent.performance.emotion = Math.trunc(now() - timeStamp); } parent.analyze('End Emotion:'); @@ -186,11 +186,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): /* parent.analyze('Start GEAR:'); if (parent.config.async) { - gearRes = parent.config.face.agegenderrace.enabled ? agegenderrace.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {}; + gearRes = parent.config.face.agegenderrace.enabled ? agegenderrace.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {}; } else { parent.state = 'run:gear'; timeStamp = now(); - gearRes = parent.config.face.agegenderrace.enabled ? await agegenderrace.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : {}; + gearRes = parent.config.face.agegenderrace.enabled ? await agegenderrace.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : {}; parent.performance.emotion = Math.trunc(now() - timeStamp); } parent.analyze('End GEAR:'); @@ -199,11 +199,11 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): // run emotion, inherits face from blazeface parent.analyze('Start Description:'); if (parent.config.async) { - descRes = parent.config.face.description.enabled ? faceres.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : []; + descRes = parent.config.face.description.enabled ? faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : []; } else { parent.state = 'run:description'; timeStamp = now(); - descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].image || tf.tensor([]), parent.config, i, faces.length) : []; + descRes = parent.config.face.description.enabled ? await faceres.predict(faces[i].tensor || tf.tensor([]), parent.config, i, faces.length) : []; parent.performance.embedding = Math.trunc(now() - timeStamp); } parent.analyze('End Description:'); @@ -226,6 +226,12 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): ? Math.max(Math.abs(faces[i].annotations.leftEyeIris[3][0] - faces[i].annotations.leftEyeIris[1][0]), Math.abs(faces[i].annotations.rightEyeIris[4][1] - faces[i].annotations.rightEyeIris[2][1])) / input.shape[2] : 0; + // optionally return tensor + const tensor = parent.config.face.detector.return ? tf.squeeze(faces[i].tensor) : null; + // dispose original face tensor + tf.dispose(faces[i].tensor); + // delete temp face image + if (faces[i].tensor) delete faces[i].tensor; // combine results faceRes.push({ ...faces[i], @@ -237,12 +243,8 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): emotion: emotionRes, iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0, rotation, - tensor: parent.config.face.detector.return ? tf.squeeze(faces[i].image) : null, + tensor, }); - // dispose original face tensor - tf.dispose(faces[i].image); - // delete temp face image - if (faces[i].image) delete faces[i].image; parent.analyze('End Face'); } diff --git a/src/handpose/handdetector.ts b/src/handpose/handdetector.ts index 88813eae..e0ce14d4 100644 --- a/src/handpose/handdetector.ts +++ b/src/handpose/handdetector.ts @@ -50,7 +50,7 @@ export class HandDetector { const boxes = this.normalizeBoxes(rawBoxes); tf.dispose(rawBoxes); const filteredT = await tf.image.nonMaxSuppressionAsync(boxes, scores, config.hand.maxDetected, config.hand.iouThreshold, config.hand.minConfidence); - const filtered = filteredT.arraySync(); + const filtered = await filteredT.array(); tf.dispose(scoresT); tf.dispose(filteredT); @@ -81,7 +81,7 @@ export class HandDetector { const boxes = prediction.box.dataSync(); const startPoint = boxes.slice(0, 2); const endPoint = boxes.slice(2, 4); - const palmLandmarks = prediction.palmLandmarks.arraySync(); + const palmLandmarks = await prediction.palmLandmarks.array(); tf.dispose(prediction.box); tf.dispose(prediction.palmLandmarks); hands.push(box.scaleBoxCoordinates({ startPoint, endPoint, palmLandmarks, confidence: prediction.confidence }, [inputWidth / this.inputSize, inputHeight / this.inputSize])); diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index 5264d56a..511733f3 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -122,7 +122,7 @@ export class HandPipeline { tf.dispose(confidenceT); if (confidence >= config.hand.minConfidence) { const keypointsReshaped = tf.reshape(keypoints, [-1, 3]); - const rawCoords = keypointsReshaped.arraySync(); + const rawCoords = await keypointsReshaped.array(); tf.dispose(keypoints); tf.dispose(keypointsReshaped); const coords = this.transformRawCoords(rawCoords, newBox, angle, rotationMatrix); diff --git a/src/movenet/movenet.ts b/src/movenet/movenet.ts index eb4f1cb6..0db93490 100644 --- a/src/movenet/movenet.ts +++ b/src/movenet/movenet.ts @@ -50,7 +50,7 @@ export async function predict(image: Tensor, config: Config): Promise { if (resT) { keypoints.length = 0; - const res = resT.arraySync(); + const res = await resT.array(); tf.dispose(resT); const kpt = res[0][0]; for (let id = 0; id < kpt.length; id++) { diff --git a/src/object/centernet.ts b/src/object/centernet.ts index 2af54dbd..4942d13a 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -28,15 +28,15 @@ export async function load(config: Config): Promise { async function process(res: Tensor, inputSize, outputShape, config: Config) { if (!res) return []; const results: Array = []; - const detections = res.arraySync(); + const detections = await res.array(); const squeezeT = tf.squeeze(res); tf.dispose(res); const arr = tf.split(squeezeT, 6, 1); // x1, y1, x2, y2, score, class tf.dispose(squeezeT); const stackT = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x - const boxesT = stackT.squeeze(); - const scoresT = arr[4].squeeze(); - const classesT = arr[5].squeeze(); + const boxesT = tf.squeeze(stackT); + const scoresT = tf.squeeze(arr[4]); + const classesT = tf.squeeze(arr[5]); arr.forEach((t) => tf.dispose(t)); const nmsT = await tf.image.nonMaxSuppressionAsync(boxesT, scoresT, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence); tf.dispose(boxesT); diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index 5219f4ee..11b3800c 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -32,14 +32,14 @@ async function process(res, inputSize, outputShape, config) { let results: Array = []; for (const strideSize of [1, 2, 4]) { // try each stride size as it detects large/medium/small objects // find scores, boxes, classes - tf.tidy(() => { // wrap in tidy to automatically deallocate temp tensors + tf.tidy(async () => { // wrap in tidy to automatically deallocate temp tensors const baseSize = strideSize * 13; // 13x13=169, 26x26=676, 52x52=2704 // find boxes and scores output depending on stride const scoresT = res.find((a) => (a.shape[1] === (baseSize ** 2) && a.shape[2] === labels.length))?.squeeze(); const featuresT = res.find((a) => (a.shape[1] === (baseSize ** 2) && a.shape[2] < labels.length))?.squeeze(); const boxesMax = featuresT.reshape([-1, 4, featuresT.shape[1] / 4]); // reshape [output] to [4, output / 4] where number is number of different features inside each stride - const boxIdx = boxesMax.argMax(2).arraySync(); // what we need is indexes of features with highest scores, not values itself - const scores = scoresT.arraySync(); // optionally use exponential scores or just as-is + const boxIdx = await boxesMax.argMax(2).array(); // what we need is indexes of features with highest scores, not values itself + const scores = await scoresT.array(); // optionally use exponential scores or just as-is for (let i = 0; i < scoresT.shape[0]; i++) { // total strides (x * y matrix) for (let j = 0; j < scoresT.shape[1]; j++) { // one score for each class const score = scores[i][j]; // get score for current position diff --git a/src/result.ts b/src/result.ts index 11babac2..e8806be7 100644 --- a/src/result.ts +++ b/src/result.ts @@ -53,8 +53,7 @@ export interface Face { matrix: [number, number, number, number, number, number, number, number, number], gaze: { bearing: number, strength: number }, } - image?: Tensor; - tensor: Tensor, + tensor?: Tensor, } /** Body results diff --git a/tfjs/tf-browser.ts b/tfjs/tf-browser.ts index 6db3d96e..b21c1eb1 100644 --- a/tfjs/tf-browser.ts +++ b/tfjs/tf-browser.ts @@ -3,7 +3,6 @@ * @external */ -// import from src // get versions of all packages import { version as tfjsVersion } from '@tensorflow/tfjs/package.json'; import { version as tfjsCoreVersion } from '@tensorflow/tfjs-core/package.json'; @@ -14,7 +13,7 @@ import { version as tfjsBackendCPUVersion } from '@tensorflow/tfjs-backend-cpu/p import { version as tfjsBackendWebGLVersion } from '@tensorflow/tfjs-backend-webgl/package.json'; import { version as tfjsBackendWASMVersion } from '@tensorflow/tfjs-backend-wasm/package.json'; -// export all +// export all from sources // requires treeShaking:ignore-annotations due to tfjs misconfiguration /* export * from '@tensorflow/tfjs-core/src/index'; @@ -26,7 +25,7 @@ export * from '@tensorflow/tfjs-backend-webgl/src/index'; export * from '@tensorflow/tfjs-backend-wasm/src/index'; */ -// export all +// export all from build export * from '@tensorflow/tfjs-core/dist/index.js'; export * from '@tensorflow/tfjs-layers/dist/index.js'; export * from '@tensorflow/tfjs-converter/dist/index.js'; @@ -34,6 +33,7 @@ export * as data from '@tensorflow/tfjs-data/dist/index.js'; export * from '@tensorflow/tfjs-backend-cpu/dist/index.js'; export * from '@tensorflow/tfjs-backend-webgl/dist/index.js'; export * from '@tensorflow/tfjs-backend-wasm/dist/index.js'; +// export * from '@tensorflow/tfjs-backend-webgpu/dist/index.js'; // experimental // export versions export const version = { @@ -46,4 +46,3 @@ export const version = { 'tfjs-backend-webgl': tfjsBackendWebGLVersion, 'tfjs-backend-wasm': tfjsBackendWASMVersion, }; -// export const version = {}; diff --git a/wiki b/wiki index 2135debf..bdc4077a 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2135debf198b5b0ecb670896bef837cbb45fe32e +Subproject commit bdc4077a3df07abdf4a2d5b2d2beadf2e573e8d8