ambiser
/
human
mirror of https://github.com/vladmandic/human
9
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

placeholder for face contours

pull/293/head
Vladimir Mandic 2022-07-13 12:08:23 -04:00
parent ccfd5ef49e
commit c08395433e
10 changed files with 739 additions and 713 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@
### **HEAD -> main** 2022/07/13 mandic00@live.com
- add webview support
- fix(gear): ensure gear.modelpath is used for loadmodel()
- npm default install should be prod only
- fix npm v7 compatibility

2
demo/index.js
View File

@ -56,10 +56,12 @@ let userConfig = {
// body: { enabled: true, modelPath: 'movenet-multipose.json' },
segmentation: { enabled: false },
*/
/*
face: { iris: { enabled: false }, emotion: { enabled: false } },
hand: { enabled: false },
body: { enabled: false },
gesture: { enabled: false },
*/
};
const drawOptions = {

7
src/draw/face.ts
View File

@ -120,6 +120,13 @@ function drawFacePolygons(f: FaceResult, ctx: CanvasRenderingContext2D | Offscre
}
drawIrisElipse(f, ctx);
}
/*
if (opt.drawPolygons && f.contours.length > 1) {
ctx.lineWidth = 5;
lines(ctx, f.contours, opt);
}
ctx.lineWidth = 1;
*/
}
function drawFacePoints(f: FaceResult, ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) {

2
src/draw/primitives.ts
View File

@ -57,7 +57,7 @@ export function lines(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingCo
ctx.beginPath();
ctx.moveTo(points[0][0], points[0][1]);
for (const pt of points) {
ctx.strokeStyle = colorDepth(pt[2], localOptions);
ctx.strokeStyle = colorDepth(pt[2] || 0, localOptions);
ctx.lineTo(Math.trunc(pt[0]), Math.trunc(pt[1]));
}
ctx.stroke();

4
src/face/blazeface.ts
View File

@ -34,7 +34,7 @@ export async function load(config: Config): Promise<GraphModel> {
return model;
}
function decodeBounds(boxOutputs: Tensor) {
function decodeBoxes(boxOutputs: Tensor) {
const t: Record<string, Tensor> = {};
t.boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
t.centers = tf.add(t.boxStarts, anchors);
@ -71,7 +71,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
t.batch = tf.squeeze(res);
}
tf.dispose(res);
t.boxes = decodeBounds(t.batch);
t.boxes = decodeBoxes(t.batch);
t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
t.sigmoid = tf.sigmoid(t.logits);
t.scores = tf.squeeze(t.sigmoid);

12
src/face/facemesh.ts
View File

@ -59,6 +59,8 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
score: 0,
boxScore: 0,
faceScore: 0,
// contoursRaw: [],
// contours: [],
annotations: {} as Record<FaceLandmark, Point[]>,
};
@ -89,6 +91,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
const confidenceT = results.find((t) => t.shape[t.shape.length - 1] === 1) as Tensor;
const faceConfidence = await confidenceT.data();
face.faceScore = Math.round(100 * faceConfidence[0]) / 100;
if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh
box.confidence = face.faceScore; // reset confidence of cached box
if (config.face.mesh?.keepInvalid) {
@ -99,7 +102,7 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
((box.startPoint[0] + box.endPoint[0])) / 2 + ((box.endPoint[0] + box.startPoint[0]) * pt[0] / blazeface.size()),
((box.startPoint[1] + box.endPoint[1])) / 2 + ((box.endPoint[1] + box.startPoint[1]) * pt[1] / blazeface.size()),
]);
face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]);
face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 1), pt[1] / (input.shape[1] || 1), (pt[2] || 0) / inputSize]);
for (const key of Object.keys(coords.blazeFaceLandmarks)) {
face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations
}
@ -121,6 +124,13 @@ export async function predict(input: Tensor, config: Config): Promise<FaceResult
const calculatedBox = { ...util.calculateFaceBox(face.mesh, box), confidence: box.confidence, landmarks: box.landmarks };
face.box = util.clampBox(calculatedBox, input);
face.boxRaw = util.getRawBox(calculatedBox, input);
/*
const contoursT = results.find((t) => t.shape[t.shape.length - 1] === 266) as Tensor;
const contoursData = contoursT && await contoursT.data(); // 133 x 2d points
face.contoursRaw = [];
for (let j = 0; j < contoursData.length / 2; j++) face.contoursRaw.push([contoursData[2 * j + 0] / inputSize, contoursData[2 * j + 1] / inputSize]);
face.contours = face.contoursRaw.map((c) => [Math.trunc((input.shape[2] || 1) * c[0]), Math.trunc((input.shape[1] || 1) * c[1])]);
*/
newCache.push(calculatedBox);
}
tf.dispose(results);

6
src/result.ts
View File

@ -40,7 +40,11 @@ export interface FaceResult {
/** detected face mesh */
mesh: Array<Point>
/** detected face mesh normalized to 0..1 */
meshRaw: Array<Point>
meshRaw: Array<Point>,
/** face contours as array of 2d points normalized to 0..1 */
// contoursRaw: Array<[number, number]>,
/** face contours as array of 2d points */
// contours: Array<[number, number]>,
/** mesh keypoints combined into annotated results */
annotations: Record<FaceLandmark, Point[]>,
/** detected age */

48
test/build.log
View File

@ -1,24 +1,24 @@
2022-07-13 09:15:44 INFO:  Application: {"name":"@vladmandic/human","version":"2.8.1"}
2022-07-13 09:15:44 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-07-13 09:15:44 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.22.18","eslint":"8.19.0"}
2022-07-13 09:15:44 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-07-13 09:15:44 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-07-13 09:15:44 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608}
2022-07-13 09:15:44 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":73,"inputBytes":644542,"outputBytes":301481}
2022-07-13 09:15:44 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612}
2022-07-13 09:15:45 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":73,"inputBytes":644546,"outputBytes":301485}
2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664}
2022-07-13 09:15:45 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":73,"inputBytes":644598,"outputBytes":301535}
2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358}
2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583}
2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":73,"inputBytes":644517,"outputBytes":300369}
2022-07-13 09:15:45 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1353528}
2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":73,"inputBytes":1997462,"outputBytes":1652983}
2022-07-13 09:15:45 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":73,"inputBytes":1997462,"outputBytes":2140437}
2022-07-13 09:15:51 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":116}
2022-07-13 09:15:54 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":74,"generated":true}
2022-07-13 09:15:54 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6324,"outputBytes":3057}
2022-07-13 09:15:54 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-07-13 09:16:02 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0}
2022-07-13 09:16:03 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-07-13 09:16:03 INFO:  Done...
2022-07-13 12:06:43 INFO:  Application: {"name":"@vladmandic/human","version":"2.8.1"}
2022-07-13 12:06:43 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
2022-07-13 12:06:43 INFO:  Toolchain: {"build":"0.7.3","esbuild":"0.14.49","typescript":"4.7.4","typedoc":"0.22.18","eslint":"8.19.0"}
2022-07-13 12:06:43 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
2022-07-13 12:06:43 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":608}
2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":73,"inputBytes":645458,"outputBytes":301484}
2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":612}
2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":73,"inputBytes":645462,"outputBytes":301488}
2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":664}
2022-07-13 12:06:43 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":73,"inputBytes":645514,"outputBytes":301538}
2022-07-13 12:06:43 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1069,"outputBytes":358}
2022-07-13 12:06:44 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1032,"outputBytes":583}
2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":73,"inputBytes":645433,"outputBytes":300372}
2022-07-13 12:06:44 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1364648}
2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":73,"inputBytes":2009498,"outputBytes":1664353}
2022-07-13 12:06:44 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":73,"inputBytes":2009498,"outputBytes":2154326}
2022-07-13 12:06:50 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":116}
2022-07-13 12:06:52 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":74,"generated":true}
2022-07-13 12:06:52 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":6324,"outputBytes":3057}
2022-07-13 12:06:52 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820}
2022-07-13 12:07:01 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":106,"errors":0,"warnings":0}
2022-07-13 12:07:02 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
2022-07-13 12:07:02 INFO:  Done...

1368
test/test.log
View File

File diff suppressed because it is too large Load Diff

2
types/human.d.ts vendored
View File

@ -601,6 +601,8 @@ export declare interface FaceResult {
mesh: Array<Point>;
/** detected face mesh normalized to 0..1 */
meshRaw: Array<Point>;
/** face contours as array of 2d points normalized to 0..1 */
/** face contours as array of 2d points */
/** mesh keypoints combined into annotated results */
annotations: Record<FaceLandmark, Point[]>;
/** detected age */