diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee5fde6..8fa3a7cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,15 @@ Repository: **** ## Changelog -### **HEAD -> main** 2021/05/30 mandic00@live.com +### **HEAD -> main** 2021/06/01 mandic00@live.com +- breaking changes to results.object output properties +- breaking changes to results.hand output properties +- breaking changes to results.body output properties + +### **origin/main** 2021/05/31 mandic00@live.com + +- implemented human.next global interpolation method - finished draw buffering and smoothing and enabled by default - implemented service worker - quantized centernet diff --git a/TODO.md b/TODO.md index 23daed60..73e6728a 100644 --- a/TODO.md +++ b/TODO.md @@ -4,18 +4,13 @@ N/A -## Exploring Features - -- Switch to TypeScript 4.3 -- Unify score/confidence variables - ## Explore Models - InsightFace: RetinaFace detector and ArcFace recognition: ## In Progress -N/A +- Switch to TypeScript 4.3 ## Known Issues @@ -23,3 +18,4 @@ N/A - NanoDet with WASM: - BlazeFace and HandPose rotation in NodeJS: - TypeDoc with TypeScript 4.3: +- HandPose lower precision with WASM diff --git a/demo/facematch.js b/demo/facematch.js index 806da022..9bc93b9f 100644 --- a/demo/facematch.js +++ b/demo/facematch.js @@ -101,7 +101,7 @@ async function analyze(face) { ctx.fillStyle = 'rgba(255, 255, 255, 1)'; ctx.fillText(`${(100 * similarity).toFixed(1)}%`, 4, 24); ctx.font = 'small-caps 0.8rem "Lato"'; - ctx.fillText(`${current.age}y ${(100 * (current.genderConfidence || 0)).toFixed(1)}% ${current.gender}`, 4, canvas.height - 6); + ctx.fillText(`${current.age}y ${(100 * (current.genderScore || 0)).toFixed(1)}% ${current.gender}`, 4, canvas.height - 6); // identify person ctx.font = 'small-caps 1rem "Lato"'; const person = await human.match(current.embedding, db); @@ -136,7 +136,7 @@ async function faces(index, res, fileName) { const ctx = canvas.getContext('2d'); ctx.font = 'small-caps 0.8rem "Lato"'; ctx.fillStyle = 'rgba(255, 255, 255, 1)'; - ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderConfidence || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6); + ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6); const person = await human.match(res.face[i].embedding, db); ctx.font = 'small-caps 1rem "Lato"'; if (person.similarity && person.similarity > minScore && res.face[i].confidence > minConfidence) ctx.fillText(`${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30); diff --git a/demo/index.js b/demo/index.js index 90eed78c..2ee43fc8 100644 --- a/demo/index.js +++ b/demo/index.js @@ -475,7 +475,7 @@ async function processImage(input) { thumb.width = window.innerWidth / (ui.columns + 0.1); thumb.height = thumb.width * canvas.height / canvas.width; if (result.face && result.face.length > 0) { - thumb.title = result.face.map((a, i) => `#${i} face: ${Math.trunc(100 * a.faceConfidence)}% box: ${Math.trunc(100 * a.boxConfidence)}% age: ${Math.trunc(a.age)} gender: ${Math.trunc(100 * a.genderConfidence)}% ${a.gender}`).join(' | '); + thumb.title = result.face.map((a, i) => `#${i} face: ${Math.trunc(100 * a.faceScore)}% box: ${Math.trunc(100 * a.boxScore)}% age: ${Math.trunc(a.age)} gender: ${Math.trunc(100 * a.genderScore)}% ${a.gender}`).join(' | '); } else { thumb.title = 'no face detected'; } diff --git a/demo/node.js b/demo/node.js index 61aaf9e3..6a09eee2 100644 --- a/demo/node.js +++ b/demo/node.js @@ -105,7 +105,7 @@ async function detect(input) { for (let i = 0; i < result.face.length; i++) { const face = result.face[i]; const emotion = face.emotion.reduce((prev, curr) => (prev.score > curr.score ? prev : curr)); - log.data(` Face: #${i} boxConfidence:${face.boxConfidence} faceConfidence:${face.faceConfidence} age:${face.age} genderConfidence:${face.genderConfidence} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} iris:${face.iris}`); + log.data(` Face: #${i} boxScore:${face.boxScore} faceScore:${face.faceScore} age:${face.age} genderScore:${face.genderScore} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} iris:${face.iris}`); } } else { log.data(' Face: N/A'); @@ -113,7 +113,7 @@ async function detect(input) { if (result && result.body && result.body.length > 0) { for (let i = 0; i < result.body.length; i++) { const body = result.body[i]; - log.data(` Body: #${i} score:${body.score} landmarks:${body.keypoints?.length || body.landmarks?.length}`); + log.data(` Body: #${i} score:${body.score} keypoints:${body.keypoints?.length}`); } } else { log.data(' Body: N/A'); @@ -121,7 +121,7 @@ async function detect(input) { if (result && result.hand && result.hand.length > 0) { for (let i = 0; i < result.hand.length; i++) { const hand = result.hand[i]; - log.data(` Hand: #${i} confidence:${hand.confidence}`); + log.data(` Hand: #${i} score:${hand.score}`); } } else { log.data(' Hand: N/A'); @@ -143,16 +143,20 @@ async function detect(input) { log.data(' Object: N/A'); } - fs.writeFileSync('result.json', JSON.stringify(result, null, 2)); // print data to console if (result) { - log.data('Persons:'); + // invoke persons getter const persons = result.persons; + + // write result objects to file + // fs.writeFileSync('result.json', JSON.stringify(result, null, 2)); + + log.data('Persons:'); for (let i = 0; i < persons.length; i++) { const face = persons[i].face; - const faceTxt = face ? `confidence:${face.confidence} age:${face.age} gender:${face.gender} iris:${face.iris}` : null; + const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null; const body = persons[i].body; - const bodyTxt = body ? `confidence:${body.score} landmarks:${body.keypoints?.length}` : null; + const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null; log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`); } } diff --git a/package.json b/package.json index 4da3536f..b5ce062f 100644 --- a/package.json +++ b/package.json @@ -63,8 +63,8 @@ "@tensorflow/tfjs-node": "^3.6.1", "@tensorflow/tfjs-node-gpu": "^3.6.1", "@types/node": "^15.6.1", - "@typescript-eslint/eslint-plugin": "^4.25.0", - "@typescript-eslint/parser": "^4.25.0", + "@typescript-eslint/eslint-plugin": "^4.26.0", + "@typescript-eslint/parser": "^4.26.0", "@vladmandic/pilogger": "^0.2.17", "canvas": "^2.8.0", "chokidar": "^3.5.1", diff --git a/src/draw/draw.ts b/src/draw/draw.ts index ffb9240f..a9f453d9 100644 --- a/src/draw/draw.ts +++ b/src/draw/draw.ts @@ -174,13 +174,13 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array, dra if (localOptions.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions); // silly hack since fillText does not suport new line const labels:string[] = []; - labels.push(`face confidence: ${Math.trunc(100 * f.score)}%`); - if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}% confident`); - // if (f.genderConfidence) labels.push(f.gender); + labels.push(`face: ${Math.trunc(100 * f.score)}%`); + if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}%`); if (f.age) labels.push(`age: ${f.age || ''}`); if (f.iris) labels.push(`distance: ${f.iris}`); if (f.emotion && f.emotion.length > 0) { const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); + if (emotion.length > 3) emotion.length = 3; labels.push(emotion.join(' ')); } if (f.rotation && f.rotation.angle && f.rotation.gaze) { diff --git a/src/efficientpose/efficientpose.ts b/src/efficientpose/efficientpose.ts index e05bab24..b00bd277 100644 --- a/src/efficientpose/efficientpose.ts +++ b/src/efficientpose/efficientpose.ts @@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types'; let model: GraphModel; -type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } }; +type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] }; const keypoints: Array = []; let box: [number, number, number, number] = [0, 0, 0, 0]; @@ -84,30 +84,30 @@ export async function predict(image, config): Promise { keypoints.push({ score: Math.round(100 * partScore) / 100, part: bodyParts[id], - positionRaw: { // normalized to 0..1 + positionRaw: [ // normalized to 0..1 // @ts-ignore model is not undefined here - x: x / model.inputs[0].shape[2], y: y / model.inputs[0].shape[1], - }, - position: { // normalized to input image size + x / model.inputs[0].shape[2], y / model.inputs[0].shape[1], + ], + position: [ // normalized to input image size // @ts-ignore model is not undefined here - x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]), - }, + Math.round(image.shape[2] * x / model.inputs[0].shape[2]), Math.round(image.shape[1] * y / model.inputs[0].shape[1]), + ], }); } } stack.forEach((s) => tf.dispose(s)); } score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0); - const x = keypoints.map((a) => a.position.x); - const y = keypoints.map((a) => a.position.y); + const x = keypoints.map((a) => a.position[0]); + const y = keypoints.map((a) => a.position[1]); box = [ Math.min(...x), Math.min(...y), Math.max(...x) - Math.min(...x), Math.max(...y) - Math.min(...y), ]; - const xRaw = keypoints.map((a) => a.positionRaw.x); - const yRaw = keypoints.map((a) => a.positionRaw.y); + const xRaw = keypoints.map((a) => a.positionRaw[0]); + const yRaw = keypoints.map((a) => a.positionRaw[1]); boxRaw = [ Math.min(...xRaw), Math.min(...yRaw), diff --git a/src/face.ts b/src/face.ts index 7f78799e..0fee78fd 100644 --- a/src/face.ts +++ b/src/face.ts @@ -160,6 +160,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): parent.analyze('Get Face'); // is something went wrong, skip the face + // @ts-ignore possibly undefined if (!faces[i].image || faces[i].image['isDisposedInternal']) { log('Face object is disposed:', faces[i].image); continue; @@ -210,12 +211,13 @@ export const detectFace = async (parent /* instance of human */, input: Tensor): : 0; // combine results + if (faces[i].image) delete faces[i].image; faceRes.push({ ...faces[i], id: i, age: descRes.age, gender: descRes.gender, - genderScore: descRes.genderConfidence, + genderScore: descRes.genderScore, embedding: descRes.descriptor, emotion: emotionRes, iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0, diff --git a/src/faceres/faceres.ts b/src/faceres/faceres.ts index 74b7431e..a2b36ebb 100644 --- a/src/faceres/faceres.ts +++ b/src/faceres/faceres.ts @@ -9,7 +9,13 @@ import * as tf from '../../dist/tfjs.esm.js'; import { Tensor, GraphModel } from '../tfjs/types'; let model: GraphModel; -const last: Array<{ age: number}> = []; +const last: Array<{ + age: number, + gender: string, + genderScore: number, + descriptor: number[], +}> = []; + let lastCount = 0; let skipped = Number.MAX_SAFE_INTEGER; @@ -108,7 +114,7 @@ export async function predict(image, config, idx, count) { if (!model) return null; if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { skipped++; - return last; + return last[idx]; } skipped = 0; return new Promise(async (resolve) => { @@ -118,8 +124,9 @@ export async function predict(image, config, idx, count) { const obj = { age: 0, gender: 'unknown', - genderConfidence: 0, - descriptor: [] }; + genderScore: 0, + descriptor: [], + }; if (config.face.description.enabled) resT = await model.predict(enhanced); tf.dispose(enhanced); @@ -130,7 +137,7 @@ export async function predict(image, config, idx, count) { const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; if (confidence > config.face.description.minConfidence) { obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; - obj.genderConfidence = Math.min(0.99, confidence); + obj.genderScore = Math.min(0.99, confidence); } const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0]; const all = resT.find((t) => t.shape[1] === 100).dataSync(); diff --git a/src/handpose/handpipeline.ts b/src/handpose/handpipeline.ts index f95d655b..7851d047 100644 --- a/src/handpose/handpipeline.ts +++ b/src/handpose/handpipeline.ts @@ -73,9 +73,9 @@ export class HandPipeline { util.dot(boxCenter, inverseRotationMatrix[1]), ]; return coordsRotated.map((coord) => [ - coord[0] + originalBoxCenter[0], - coord[1] + originalBoxCenter[1], - coord[2], + Math.trunc(coord[0] + originalBoxCenter[0]), + Math.trunc(coord[1] + originalBoxCenter[1]), + Math.trunc(coord[2]), ]); } diff --git a/src/handpose/handpose.ts b/src/handpose/handpose.ts index 8462b16b..df4aff83 100644 --- a/src/handpose/handpose.ts +++ b/src/handpose/handpose.ts @@ -51,10 +51,10 @@ export async function predict(input, config): Promise { boxRaw = [box[0] / input.shape[2], box[1] / input.shape[1], box[2] / input.shape[2], box[3] / input.shape[1]]; } else { // otherwise use box from prediction box = predictions[i].box ? [ - Math.max(0, predictions[i].box.topLeft[0]), - Math.max(0, predictions[i].box.topLeft[1]), - Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0]), - Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1]), + Math.trunc(Math.max(0, predictions[i].box.topLeft[0])), + Math.trunc(Math.max(0, predictions[i].box.topLeft[1])), + Math.trunc(Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0])), + Math.trunc(Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1])), ] : [0, 0, 0, 0]; boxRaw = [ (predictions[i].box.topLeft[0]) / input.shape[2], diff --git a/src/result.ts b/src/result.ts index 3a024fb7..14e9bc84 100644 --- a/src/result.ts +++ b/src/result.ts @@ -52,7 +52,7 @@ export interface Face { matrix: [number, number, number, number, number, number, number, number, number], gaze: { bearing: number, strength: number }, } - image: typeof Tensor; + image?: typeof Tensor; tensor: typeof Tensor, } diff --git a/wiki b/wiki index f0b3ba94..bc7cb668 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f0b3ba9432ba6ca2ed35c09b6193fa685cb3bced +Subproject commit bc7cb66846f150664df61777dea298bdc99492b2