ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

release candidate

pull/280/head
Vladimir Mandic 2021-06-01 08:59:09 -04:00
parent e19c65526c
commit b79e380114
14 changed files with 64 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -9,8 +9,15 @@ Repository: **<git+https://github.com/vladmandic/human.git>**
## Changelog
### **HEAD -> main** 2021/05/30 mandic00@live.com
### **HEAD -> main** 2021/06/01 mandic00@live.com
- breaking changes to results.object output properties
- breaking changes to results.hand output properties
- breaking changes to results.body output properties
### **origin/main** 2021/05/31 mandic00@live.com
- implemented human.next global interpolation method
- finished draw buffering and smoothing and enabled by default
- implemented service worker
- quantized centernet

8
TODO.md
View File

@ -4,18 +4,13 @@
N/A
## Exploring Features
- Switch to TypeScript 4.3
- Unify score/confidence variables
## Explore Models
- InsightFace: RetinaFace detector and ArcFace recognition: <https://github.com/deepinsight/insightface>
## In Progress
N/A
- Switch to TypeScript 4.3
## Known Issues
@ -23,3 +18,4 @@ N/A
- NanoDet with WASM: <https://github.com/tensorflow/tfjs/issues/4824>
- BlazeFace and HandPose rotation in NodeJS: <https://github.com/tensorflow/tfjs/issues/4066>
- TypeDoc with TypeScript 4.3: <https://github.com/TypeStrong/typedoc/issues/1589>
- HandPose lower precision with WASM

4
demo/facematch.js
View File

@ -101,7 +101,7 @@ async function analyze(face) {
ctx.fillStyle = 'rgba(255, 255, 255, 1)';
ctx.fillText(`${(100 * similarity).toFixed(1)}%`, 4, 24);
ctx.font = 'small-caps 0.8rem "Lato"';
ctx.fillText(`${current.age}y ${(100 * (current.genderConfidence || 0)).toFixed(1)}% ${current.gender}`, 4, canvas.height - 6);
ctx.fillText(`${current.age}y ${(100 * (current.genderScore || 0)).toFixed(1)}% ${current.gender}`, 4, canvas.height - 6);
// identify person
ctx.font = 'small-caps 1rem "Lato"';
const person = await human.match(current.embedding, db);
@ -136,7 +136,7 @@ async function faces(index, res, fileName) {
const ctx = canvas.getContext('2d');
ctx.font = 'small-caps 0.8rem "Lato"';
ctx.fillStyle = 'rgba(255, 255, 255, 1)';
ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderConfidence || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
ctx.fillText(`${res.face[i].age}y ${(100 * (res.face[i].genderScore || 0)).toFixed(1)}% ${res.face[i].gender}`, 4, canvas.height - 6);
const person = await human.match(res.face[i].embedding, db);
ctx.font = 'small-caps 1rem "Lato"';
if (person.similarity && person.similarity > minScore && res.face[i].confidence > minConfidence) ctx.fillText(`${(100 * person.similarity).toFixed(1)}% ${person.name}`, 4, canvas.height - 30);

2
demo/index.js
View File

@ -475,7 +475,7 @@ async function processImage(input) {
thumb.width = window.innerWidth / (ui.columns + 0.1);
thumb.height = thumb.width * canvas.height / canvas.width;
if (result.face && result.face.length > 0) {
thumb.title = result.face.map((a, i) => `#${i} face: ${Math.trunc(100 * a.faceConfidence)}% box: ${Math.trunc(100 * a.boxConfidence)}% age: ${Math.trunc(a.age)} gender: ${Math.trunc(100 * a.genderConfidence)}% ${a.gender}`).join(' | ');
thumb.title = result.face.map((a, i) => `#${i} face: ${Math.trunc(100 * a.faceScore)}% box: ${Math.trunc(100 * a.boxScore)}% age: ${Math.trunc(a.age)} gender: ${Math.trunc(100 * a.genderScore)}% ${a.gender}`).join(' | ');
} else {
thumb.title = 'no face detected';
}

18
demo/node.js
View File

@ -105,7 +105,7 @@ async function detect(input) {
for (let i = 0; i < result.face.length; i++) {
const face = result.face[i];
const emotion = face.emotion.reduce((prev, curr) => (prev.score > curr.score ? prev : curr));
log.data(` Face: #${i} boxConfidence:${face.boxConfidence} faceConfidence:${face.faceConfidence} age:${face.age} genderConfidence:${face.genderConfidence} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} iris:${face.iris}`);
log.data(` Face: #${i} boxScore:${face.boxScore} faceScore:${face.faceScore} age:${face.age} genderScore:${face.genderScore} gender:${face.gender} emotionScore:${emotion.score} emotion:${emotion.emotion} iris:${face.iris}`);
}
} else {
log.data(' Face: N/A');
@ -113,7 +113,7 @@ async function detect(input) {
if (result && result.body && result.body.length > 0) {
for (let i = 0; i < result.body.length; i++) {
const body = result.body[i];
log.data(` Body: #${i} score:${body.score} landmarks:${body.keypoints?.length || body.landmarks?.length}`);
log.data(` Body: #${i} score:${body.score} keypoints:${body.keypoints?.length}`);
}
} else {
log.data(' Body: N/A');
@ -121,7 +121,7 @@ async function detect(input) {
if (result && result.hand && result.hand.length > 0) {
for (let i = 0; i < result.hand.length; i++) {
const hand = result.hand[i];
log.data(` Hand: #${i} confidence:${hand.confidence}`);
log.data(` Hand: #${i} score:${hand.score}`);
}
} else {
log.data(' Hand: N/A');
@ -143,16 +143,20 @@ async function detect(input) {
log.data(' Object: N/A');
}
fs.writeFileSync('result.json', JSON.stringify(result, null, 2));
// print data to console
if (result) {
log.data('Persons:');
// invoke persons getter
const persons = result.persons;
// write result objects to file
// fs.writeFileSync('result.json', JSON.stringify(result, null, 2));
log.data('Persons:');
for (let i = 0; i < persons.length; i++) {
const face = persons[i].face;
const faceTxt = face ? `confidence:${face.confidence} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
const faceTxt = face ? `score:${face.score} age:${face.age} gender:${face.gender} iris:${face.iris}` : null;
const body = persons[i].body;
const bodyTxt = body ? `confidence:${body.score} landmarks:${body.keypoints?.length}` : null;
const bodyTxt = body ? `score:${body.score} keypoints:${body.keypoints?.length}` : null;
log.data(` #${i}: Face:${faceTxt} Body:${bodyTxt} LeftHand:${persons[i].hands.left ? 'yes' : 'no'} RightHand:${persons[i].hands.right ? 'yes' : 'no'} Gestures:${persons[i].gestures.length}`);
}
}

4
package.json
View File

@ -63,8 +63,8 @@
"@tensorflow/tfjs-node": "^3.6.1",
"@tensorflow/tfjs-node-gpu": "^3.6.1",
"@types/node": "^15.6.1",
"@typescript-eslint/eslint-plugin": "^4.25.0",
"@typescript-eslint/parser": "^4.25.0",
"@typescript-eslint/eslint-plugin": "^4.26.0",
"@typescript-eslint/parser": "^4.26.0",
"@vladmandic/pilogger": "^0.2.17",
"canvas": "^2.8.0",
"chokidar": "^3.5.1",

6
src/draw/draw.ts
View File

@ -174,13 +174,13 @@ export async function face(inCanvas: HTMLCanvasElement, result: Array<Face>, dra
if (localOptions.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions);
// silly hack since fillText does not suport new line
const labels:string[] = [];
labels.push(`face confidence: ${Math.trunc(100 * f.score)}%`);
if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}% confident`);
// if (f.genderConfidence) labels.push(f.gender);
labels.push(`face: ${Math.trunc(100 * f.score)}%`);
if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}%`);
if (f.age) labels.push(`age: ${f.age || ''}`);
if (f.iris) labels.push(`distance: ${f.iris}`);
if (f.emotion && f.emotion.length > 0) {
const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`);
if (emotion.length > 3) emotion.length = 3;
labels.push(emotion.join(' '));
}
if (f.rotation && f.rotation.angle && f.rotation.gaze) {

22
src/efficientpose/efficientpose.ts
View File

@ -9,7 +9,7 @@ import { GraphModel } from '../tfjs/types';
let model: GraphModel;
type Keypoints = { score: number, part: string, position: { x: number, y: number }, positionRaw: { x: number, y: number } };
type Keypoints = { score: number, part: string, position: [number, number], positionRaw: [number, number] };
const keypoints: Array<Keypoints> = [];
let box: [number, number, number, number] = [0, 0, 0, 0];
@ -84,30 +84,30 @@ export async function predict(image, config): Promise<Body[]> {
keypoints.push({
score: Math.round(100 * partScore) / 100,
part: bodyParts[id],
positionRaw: { // normalized to 0..1
positionRaw: [ // normalized to 0..1
// @ts-ignore model is not undefined here
x: x / model.inputs[0].shape[2], y: y / model.inputs[0].shape[1],
},
position: { // normalized to input image size
x / model.inputs[0].shape[2], y / model.inputs[0].shape[1],
],
position: [ // normalized to input image size
// @ts-ignore model is not undefined here
x: Math.round(image.shape[2] * x / model.inputs[0].shape[2]), y: Math.round(image.shape[1] * y / model.inputs[0].shape[1]),
},
Math.round(image.shape[2] * x / model.inputs[0].shape[2]), Math.round(image.shape[1] * y / model.inputs[0].shape[1]),
],
});
}
}
stack.forEach((s) => tf.dispose(s));
}
score = keypoints.reduce((prev, curr) => (curr.score > prev ? curr.score : prev), 0);
const x = keypoints.map((a) => a.position.x);
const y = keypoints.map((a) => a.position.y);
const x = keypoints.map((a) => a.position[0]);
const y = keypoints.map((a) => a.position[1]);
box = [
Math.min(...x),
Math.min(...y),
Math.max(...x) - Math.min(...x),
Math.max(...y) - Math.min(...y),
];
const xRaw = keypoints.map((a) => a.positionRaw.x);
const yRaw = keypoints.map((a) => a.positionRaw.y);
const xRaw = keypoints.map((a) => a.positionRaw[0]);
const yRaw = keypoints.map((a) => a.positionRaw[1]);
boxRaw = [
Math.min(...xRaw),
Math.min(...yRaw),

4
src/face.ts
View File

@ -160,6 +160,7 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
parent.analyze('Get Face');
// is something went wrong, skip the face
// @ts-ignore possibly undefined
if (!faces[i].image || faces[i].image['isDisposedInternal']) {
log('Face object is disposed:', faces[i].image);
continue;
@ -210,12 +211,13 @@ export const detectFace = async (parent /* instance of human */, input: Tensor):
: 0;
// combine results
if (faces[i].image) delete faces[i].image;
faceRes.push({
...faces[i],
id: i,
age: descRes.age,
gender: descRes.gender,
genderScore: descRes.genderConfidence,
genderScore: descRes.genderScore,
embedding: descRes.descriptor,
emotion: emotionRes,
iris: irisSize !== 0 ? Math.trunc(500 / irisSize / 11.7) / 100 : 0,

17
src/faceres/faceres.ts
View File

@ -9,7 +9,13 @@ import * as tf from '../../dist/tfjs.esm.js';
import { Tensor, GraphModel } from '../tfjs/types';
let model: GraphModel;
const last: Array<{ age: number}> = [];
const last: Array<{
age: number,
gender: string,
genderScore: number,
descriptor: number[],
}> = [];
let lastCount = 0;
let skipped = Number.MAX_SAFE_INTEGER;
@ -108,7 +114,7 @@ export async function predict(image, config, idx, count) {
if (!model) return null;
if ((skipped < config.face.description.skipFrames) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
skipped++;
return last;
return last[idx];
}
skipped = 0;
return new Promise(async (resolve) => {
@ -118,8 +124,9 @@ export async function predict(image, config, idx, count) {
const obj = {
age: <number>0,
gender: <string>'unknown',
genderConfidence: <number>0,
descriptor: <number[]>[] };
genderScore: <number>0,
descriptor: <number[]>[],
};
if (config.face.description.enabled) resT = await model.predict(enhanced);
tf.dispose(enhanced);
@ -130,7 +137,7 @@ export async function predict(image, config, idx, count) {
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
if (confidence > config.face.description.minConfidence) {
obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
obj.genderConfidence = Math.min(0.99, confidence);
obj.genderScore = Math.min(0.99, confidence);
}
const age = resT.find((t) => t.shape[1] === 100).argMax(1).dataSync()[0];
const all = resT.find((t) => t.shape[1] === 100).dataSync();

6
src/handpose/handpipeline.ts
View File

@ -73,9 +73,9 @@ export class HandPipeline {
util.dot(boxCenter, inverseRotationMatrix[1]),
];
return coordsRotated.map((coord) => [
coord[0] + originalBoxCenter[0],
coord[1] + originalBoxCenter[1],
coord[2],
Math.trunc(coord[0] + originalBoxCenter[0]),
Math.trunc(coord[1] + originalBoxCenter[1]),
Math.trunc(coord[2]),
]);
}

8
src/handpose/handpose.ts
View File

@ -51,10 +51,10 @@ export async function predict(input, config): Promise<Hand[]> {
boxRaw = [box[0] / input.shape[2], box[1] / input.shape[1], box[2] / input.shape[2], box[3] / input.shape[1]];
} else { // otherwise use box from prediction
box = predictions[i].box ? [
Math.max(0, predictions[i].box.topLeft[0]),
Math.max(0, predictions[i].box.topLeft[1]),
Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0]),
Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1]),
Math.trunc(Math.max(0, predictions[i].box.topLeft[0])),
Math.trunc(Math.max(0, predictions[i].box.topLeft[1])),
Math.trunc(Math.min(input.shape[2], predictions[i].box.bottomRight[0]) - Math.max(0, predictions[i].box.topLeft[0])),
Math.trunc(Math.min(input.shape[1], predictions[i].box.bottomRight[1]) - Math.max(0, predictions[i].box.topLeft[1])),
] : [0, 0, 0, 0];
boxRaw = [
(predictions[i].box.topLeft[0]) / input.shape[2],

2
src/result.ts
View File

@ -52,7 +52,7 @@ export interface Face {
matrix: [number, number, number, number, number, number, number, number, number],
gaze: { bearing: number, strength: number },
}
image: typeof Tensor;
image?: typeof Tensor;
tensor: typeof Tensor,
}

2
wiki

@ -1 +1 @@
Subproject commit f0b3ba9432ba6ca2ed35c09b6193fa685cb3bced
Subproject commit bc7cb66846f150664df61777dea298bdc99492b2