commit
f4d4780267
|
@ -93,6 +93,7 @@ async function main() {
|
||||||
log.header();
|
log.header();
|
||||||
log.info('FaceAPI single-process test');
|
log.info('FaceAPI single-process test');
|
||||||
|
|
||||||
|
// eslint-disable-next-line node/no-extraneous-import
|
||||||
fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-missing-import
|
fetch = (await import('node-fetch')).default; // eslint-disable-line node/no-missing-import
|
||||||
|
|
||||||
await faceapi.tf.setBackend('tensorflow');
|
await faceapi.tf.setBackend('tensorflow');
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
export declare const version: {
|
||||||
|
'tfjs-core': string;
|
||||||
|
'tfjs-backend-cpu': string;
|
||||||
|
'tfjs-backend-webgl': string;
|
||||||
|
'tfjs-data': string;
|
||||||
|
'tfjs-layers': string;
|
||||||
|
'tfjs-converter': string;
|
||||||
|
tfjs: string;
|
||||||
|
};
|
File diff suppressed because it is too large
Load Diff
|
@ -102,7 +102,7 @@ export abstract class NeuralNetwork<TNetParams> {
|
||||||
}
|
}
|
||||||
const { readFile } = env.getEnv();
|
const { readFile } = env.getEnv();
|
||||||
const { manifestUri, modelBaseUri } = getModelUris(filePath, this.getDefaultModelName());
|
const { manifestUri, modelBaseUri } = getModelUris(filePath, this.getDefaultModelName());
|
||||||
const fetchWeightsFromDisk = (filePaths: string[]) => Promise.all(filePaths.map((fp) => readFile(fp).then((buf) => buf.buffer)));
|
const fetchWeightsFromDisk = (filePaths: string[]) => Promise.all(filePaths.map((fp) => readFile(fp).then((buf) => (typeof buf === 'string' ? Buffer.from(buf) : buf.buffer))));
|
||||||
const loadWeights = tf['io'].weightsLoaderFactory(fetchWeightsFromDisk);
|
const loadWeights = tf['io'].weightsLoaderFactory(fetchWeightsFromDisk);
|
||||||
const manifest = JSON.parse((await readFile(manifestUri)).toString());
|
const manifest = JSON.parse((await readFile(manifestUri)).toString());
|
||||||
const weightMap = await loadWeights(manifest, modelBaseUri);
|
const weightMap = await loadWeights(manifest, modelBaseUri);
|
||||||
|
|
|
@ -13,7 +13,8 @@ export function createFileSystem(fs?: any): FileSystem {
|
||||||
}
|
}
|
||||||
|
|
||||||
const readFile = fs
|
const readFile = fs
|
||||||
? (filePath: string) => new Promise((resolve, reject) => { fs.readFile(filePath, (err: any, buffer) => (err ? reject(err) : resolve(buffer))); })
|
// eslint-disable-next-line no-undef
|
||||||
|
? (filePath: string) => new Promise<string | Buffer>((resolve, reject) => { fs.readFile(filePath, (err: NodeJS.ErrnoException | null, buffer: string | Buffer) => (err ? reject(err) : resolve(buffer))); })
|
||||||
: () => { throw new Error(`readFile - failed to require fs in nodejs environment with error: ${requireFsError}`); };
|
: () => { throw new Error(`readFile - failed to require fs in nodejs environment with error: ${requireFsError}`); };
|
||||||
return { readFile };
|
return { readFile };
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,11 +3,9 @@ import { createFileSystem } from './createFileSystem';
|
||||||
import { Environment } from './types';
|
import { Environment } from './types';
|
||||||
|
|
||||||
export function createNodejsEnv(): Environment {
|
export function createNodejsEnv(): Environment {
|
||||||
// eslint-disable-next-line dot-notation
|
const Canvas: (new () => HTMLCanvasElement) = (global as any)['Canvas'] || global.HTMLCanvasElement;
|
||||||
const Canvas = global['Canvas'] || global.HTMLCanvasElement;
|
|
||||||
const Image = global.Image || global.HTMLImageElement;
|
const Image = global.Image || global.HTMLImageElement;
|
||||||
// eslint-disable-next-line dot-notation
|
const Video: (new () => HTMLVideoElement) = (global as any)['Video'] || global.HTMLVideoElement;
|
||||||
const Video = global['Video'] || global.HTMLVideoElement;
|
|
||||||
|
|
||||||
const createCanvasElement = () => {
|
const createCanvasElement = () => {
|
||||||
if (Canvas) return new Canvas();
|
if (Canvas) return new Canvas();
|
||||||
|
|
|
@ -1,17 +1,17 @@
|
||||||
export type FileSystem = {
|
export type FileSystem = {
|
||||||
// eslint-disable-next-line no-unused-vars
|
// eslint-disable-next-line no-unused-vars
|
||||||
readFile: (filePath: string) => Promise<any>
|
readFile: (filePath: string) => Promise<string | Buffer>;
|
||||||
}
|
};
|
||||||
|
|
||||||
export type Environment = FileSystem & {
|
export type Environment = FileSystem & {
|
||||||
Canvas: typeof HTMLCanvasElement
|
Canvas: typeof HTMLCanvasElement;
|
||||||
CanvasRenderingContext2D: typeof CanvasRenderingContext2D
|
CanvasRenderingContext2D: typeof CanvasRenderingContext2D;
|
||||||
Image: typeof HTMLImageElement
|
Image: typeof HTMLImageElement;
|
||||||
ImageData: typeof ImageData
|
ImageData: typeof ImageData;
|
||||||
Video: typeof HTMLVideoElement
|
Video: typeof HTMLVideoElement;
|
||||||
createCanvasElement: () => HTMLCanvasElement
|
createCanvasElement: () => HTMLCanvasElement;
|
||||||
createImageElement: () => HTMLImageElement
|
createImageElement: () => HTMLImageElement;
|
||||||
createVideoElement: () => HTMLVideoElement
|
createVideoElement: () => HTMLVideoElement;
|
||||||
// eslint-disable-next-line no-undef, no-unused-vars
|
// eslint-disable-next-line no-undef, no-unused-vars
|
||||||
fetch: (url: string, init?: RequestInit) => Promise<Response>
|
fetch: (url: string, init?: RequestInit) => Promise<Response>;
|
||||||
}
|
};
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
export const FACE_EXPRESSION_LABELS = ['neutral', 'happy', 'sad', 'angry', 'fearful', 'disgusted', 'surprised'];
|
export const FACE_EXPRESSION_LABELS = ['neutral', 'happy', 'sad', 'angry', 'fearful', 'disgusted', 'surprised'] as const;
|
||||||
|
|
||||||
export class FaceExpressions {
|
export class FaceExpressions {
|
||||||
public neutral = 0;
|
public neutral = 0;
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { Point } from '../classes';
|
||||||
import { FaceDetection } from '../classes/FaceDetection';
|
import { FaceDetection } from '../classes/FaceDetection';
|
||||||
import { FaceLandmarks } from '../classes/FaceLandmarks';
|
import { FaceLandmarks } from '../classes/FaceLandmarks';
|
||||||
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
|
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
|
||||||
|
@ -22,20 +23,17 @@ export function isWithFaceLandmarks(
|
||||||
): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
|
): obj is WithFaceLandmarks<WithFaceDetection<{}>, FaceLandmarks> {
|
||||||
return (
|
return (
|
||||||
isWithFaceDetection(obj)
|
isWithFaceDetection(obj)
|
||||||
// eslint-disable-next-line dot-notation
|
&& (obj as any)['landmarks'] instanceof FaceLandmarks
|
||||||
&& obj['landmarks'] instanceof FaceLandmarks
|
&& (obj as any)['unshiftedLandmarks'] instanceof FaceLandmarks
|
||||||
// eslint-disable-next-line dot-notation
|
&& (obj as any)['alignedRect'] instanceof FaceDetection
|
||||||
&& obj['unshiftedLandmarks'] instanceof FaceLandmarks
|
|
||||||
// eslint-disable-next-line dot-notation
|
|
||||||
&& obj['alignedRect'] instanceof FaceDetection
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateFaceAngle(mesh) {
|
function calculateFaceAngle(mesh: FaceLandmarks) {
|
||||||
// Helper to convert radians to degrees
|
// Helper to convert radians to degrees
|
||||||
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
|
||||||
const degrees = (radians) => (radians * 180) / Math.PI;
|
const degrees = (radians: number) => (radians * 180) / Math.PI;
|
||||||
const calcLengthBetweenTwoPoints = (a, b) => Math.sqrt((a._x - b._x) ** 2 + (a._y - b._y) ** 2);
|
const calcLengthBetweenTwoPoints = (a: Point, b: Point) => Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);
|
||||||
|
|
||||||
const angle = {
|
const angle = {
|
||||||
roll: <number | undefined>undefined,
|
roll: <number | undefined>undefined,
|
||||||
|
@ -43,54 +41,51 @@ function calculateFaceAngle(mesh) {
|
||||||
yaw: <number | undefined>undefined,
|
yaw: <number | undefined>undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const calcYaw = (leftPoint, midPoint, rightPoint) => {
|
const calcYaw = (leftPoint: Point, midPoint: Point, rightPoint: Point) => {
|
||||||
// Calc x-distance from left side of the face ("ear") to facial midpoint ("nose")
|
// Calc x-distance from left side of the face ("ear") to facial midpoint ("nose")
|
||||||
const leftToMidpoint = Math.floor(leftPoint._x - midPoint._x);
|
const leftToMidpoint = Math.floor(leftPoint.x - midPoint.x);
|
||||||
// Calc x-distance from facial midpoint ("nose") to the right side of the face ("ear")
|
// Calc x-distance from facial midpoint ("nose") to the right side of the face ("ear")
|
||||||
const rightToMidpoint = Math.floor(midPoint._x - rightPoint._x);
|
const rightToMidpoint = Math.floor(midPoint.x - rightPoint.x);
|
||||||
// Difference in distances coincidentally approximates to angles
|
// Difference in distances coincidentally approximates to angles
|
||||||
return leftToMidpoint - rightToMidpoint;
|
return leftToMidpoint - rightToMidpoint;
|
||||||
};
|
};
|
||||||
|
|
||||||
const calcRoll = (lever, pivot) => {
|
const calcRoll = (lever: Point, pivot: Point) => {
|
||||||
// When rolling, the head seems to pivot from the nose/lips/chin area.
|
// When rolling, the head seems to pivot from the nose/lips/chin area.
|
||||||
// So, we'll choose any two points from the facial midline, where the first point should be the pivot, and the other "lever"
|
// So, we'll choose any two points from the facial midline, where the first point should be the pivot, and the other "lever"
|
||||||
// Plan/Execution: get the hypotenuse & opposite sides of a 90deg triangle ==> Calculate angle in radians
|
// Plan/Execution: get the hypotenuse & opposite sides of a 90deg triangle ==> Calculate angle in radians
|
||||||
const hypotenuse = Math.hypot(pivot._x - lever._x, pivot._y - lever._y);
|
const hypotenuse = Math.hypot(pivot.x - lever.x, pivot.y - lever.y);
|
||||||
const opposite = pivot._y - lever._y;
|
const opposite = pivot.y - lever.y;
|
||||||
const angleInRadians = Math.asin(opposite / hypotenuse);
|
const angleInRadians = Math.asin(opposite / hypotenuse);
|
||||||
const angleInDegrees = degrees(angleInRadians);
|
const angleInDegrees = degrees(angleInRadians);
|
||||||
const normalizeAngle = Math.floor(90 - angleInDegrees);
|
const normalizeAngle = Math.floor(90 - angleInDegrees);
|
||||||
// If lever more to the left of the pivot, then we're tilting left
|
// If lever more to the left of the pivot, then we're tilting left
|
||||||
// "-" is negative direction. "+", or absence of a sign is positive direction
|
// "-" is negative direction. "+", or absence of a sign is positive direction
|
||||||
const tiltDirection = pivot._x - lever._x < 0 ? -1 : 1;
|
const tiltDirection = pivot.x - lever.x < 0 ? -1 : 1;
|
||||||
const result = normalizeAngle * tiltDirection;
|
const result = normalizeAngle * tiltDirection;
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
const calcPitch = (leftPoint, midPoint, rightPoint) => {
|
const calcPitch = (leftPoint: Point, midPoint: Point, rightPoint: Point) => {
|
||||||
// Theory: While pitching, the nose is the most salient point --> That's what we'll use to make a trianle.
|
// Theory: While pitching, the nose is the most salient point --> That's what we'll use to make a trianle.
|
||||||
// The "base" is between point that don't move when we pitch our head (i.e. an imaginary line running ear to ear through the nose).
|
// The "base" is between point that don't move when we pitch our head (i.e. an imaginary line running ear to ear through the nose).
|
||||||
// Executuin: Get the opposite & adjacent lengths of the triangle from the ear's perspective. Use it to get angle.
|
// Executuin: Get the opposite & adjacent lengths of the triangle from the ear's perspective. Use it to get angle.
|
||||||
|
|
||||||
const base = calcLengthBetweenTwoPoints(leftPoint, rightPoint);
|
const base = calcLengthBetweenTwoPoints(leftPoint, rightPoint);
|
||||||
// adjecent is base/2 technically.
|
// adjecent is base/2 technically.
|
||||||
const baseCoords = {
|
const baseCoords = new Point((leftPoint.x + rightPoint.x) / 2, (leftPoint.y + rightPoint.y) / 2);
|
||||||
_x: (leftPoint._x + rightPoint._x) / 2,
|
|
||||||
_y: (leftPoint._y + rightPoint._y) / 2,
|
|
||||||
};
|
|
||||||
const midToBaseLength = calcLengthBetweenTwoPoints(midPoint, baseCoords);
|
const midToBaseLength = calcLengthBetweenTwoPoints(midPoint, baseCoords);
|
||||||
const angleInRadians = Math.atan(midToBaseLength / base);
|
const angleInRadians = Math.atan(midToBaseLength / base);
|
||||||
const angleInDegrees = Math.floor(degrees(angleInRadians));
|
const angleInDegrees = Math.floor(degrees(angleInRadians));
|
||||||
// Account for directionality.
|
// Account for directionality.
|
||||||
// pitch forwards (_i.e. tilting your head forwards) is positive (or no sign); backward is negative.
|
// pitch forwards (_i.e. tilting your head forwards) is positive (or no sign); backward is negative.
|
||||||
const direction = baseCoords._y - midPoint._y < 0 ? -1 : 1;
|
const direction = baseCoords.y - midPoint.y < 0 ? -1 : 1;
|
||||||
const result = angleInDegrees * direction;
|
const result = angleInDegrees * direction;
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!mesh || !mesh._positions || mesh._positions.length !== 68) return angle;
|
if (!mesh || !mesh.positions || mesh.positions.length !== 68) return angle;
|
||||||
const pt = mesh._positions;
|
const pt = mesh.positions;
|
||||||
angle.roll = calcRoll(pt[27], pt[66]);
|
angle.roll = calcRoll(pt[27], pt[66]);
|
||||||
angle.pitch = calcPitch(pt[14], pt[30], pt[2]);
|
angle.pitch = calcPitch(pt[14], pt[30], pt[2]);
|
||||||
angle.yaw = calcYaw(pt[14], pt[33], pt[2]);
|
angle.yaw = calcYaw(pt[14], pt[33], pt[2]);
|
||||||
|
|
|
@ -19,7 +19,7 @@ import { extractParams } from './extractParams';
|
||||||
import { extractParamsFromWeightMap } from './extractParamsFromWeightMap';
|
import { extractParamsFromWeightMap } from './extractParamsFromWeightMap';
|
||||||
import { leaky } from './leaky';
|
import { leaky } from './leaky';
|
||||||
import { ITinyYolov2Options, TinyYolov2Options } from './TinyYolov2Options';
|
import { ITinyYolov2Options, TinyYolov2Options } from './TinyYolov2Options';
|
||||||
import { DefaultTinyYolov2NetParams, MobilenetParams, TinyYolov2NetParams } from './types';
|
import { DefaultTinyYolov2NetParams, MobilenetParams, TinyYolov2ExtractBoxesResult, TinyYolov2NetParams } from './types';
|
||||||
|
|
||||||
export class TinyYolov2Base extends NeuralNetwork<TinyYolov2NetParams> {
|
export class TinyYolov2Base extends NeuralNetwork<TinyYolov2NetParams> {
|
||||||
public static DEFAULT_FILTER_SIZES = [3, 16, 32, 64, 128, 256, 512, 1024, 1024];
|
public static DEFAULT_FILTER_SIZES = [3, 16, 32, 64, 128, 256, 512, 1024, 1024];
|
||||||
|
@ -183,9 +183,9 @@ export class TinyYolov2Base extends NeuralNetwork<TinyYolov2NetParams> {
|
||||||
return [boxes, scores, classScores];
|
return [boxes, scores, classScores];
|
||||||
});
|
});
|
||||||
|
|
||||||
const results = [] as any;
|
const results: TinyYolov2ExtractBoxesResult[] = [];
|
||||||
const scoresData = await scoresTensor.array();
|
const scoresData = await scoresTensor.array() as number[][][][];
|
||||||
const boxesData = await boxesTensor.array();
|
const boxesData = await boxesTensor.array() as number[][][][];
|
||||||
for (let row = 0; row < numCells; row++) {
|
for (let row = 0; row < numCells; row++) {
|
||||||
for (let col = 0; col < numCells; col++) {
|
for (let col = 0; col < numCells; col++) {
|
||||||
for (let anchor = 0; anchor < numBoxes; anchor++) {
|
for (let anchor = 0; anchor < numBoxes; anchor++) {
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import * as tf from '../../dist/tfjs.esm';
|
import * as tf from '../../dist/tfjs.esm';
|
||||||
|
import { BoundingBox } from '../classes';
|
||||||
|
|
||||||
import { ConvParams } from '../common/index';
|
import { ConvParams } from '../common/index';
|
||||||
import { SeparableConvParams } from '../common/types';
|
import { SeparableConvParams } from '../common/types';
|
||||||
|
@ -38,3 +39,13 @@ export type DefaultTinyYolov2NetParams = {
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TinyYolov2NetParams = DefaultTinyYolov2NetParams | MobilenetParams
|
export type TinyYolov2NetParams = DefaultTinyYolov2NetParams | MobilenetParams
|
||||||
|
|
||||||
|
export type TinyYolov2ExtractBoxesResult = {
|
||||||
|
box: BoundingBox;
|
||||||
|
score: number;
|
||||||
|
classScore: number;
|
||||||
|
label: number;
|
||||||
|
row: number;
|
||||||
|
col: number;
|
||||||
|
anchor: number;
|
||||||
|
}
|
||||||
|
|
|
@ -56,7 +56,7 @@ export function extractParams(weights: Float32Array, numMainBlocks: number): { p
|
||||||
reduction_block_1: entry_flow_reduction_block_1,
|
reduction_block_1: entry_flow_reduction_block_1,
|
||||||
};
|
};
|
||||||
|
|
||||||
const middle_flow = {};
|
const middle_flow: Record<`main_block_${number}`, MainBlockParams> = {};
|
||||||
range(numMainBlocks, 0, 1).forEach((idx) => {
|
range(numMainBlocks, 0, 1).forEach((idx) => {
|
||||||
middle_flow[`main_block_${idx}`] = extractMainBlockParams(128, `middle_flow/main_block_${idx}`);
|
middle_flow[`main_block_${idx}`] = extractMainBlockParams(128, `middle_flow/main_block_${idx}`);
|
||||||
});
|
});
|
||||||
|
|
|
@ -58,7 +58,7 @@ export function extractParamsFromWeightMap(
|
||||||
reduction_block_1: entry_flow_reduction_block_1,
|
reduction_block_1: entry_flow_reduction_block_1,
|
||||||
};
|
};
|
||||||
|
|
||||||
const middle_flow = {};
|
const middle_flow: Record<`main_block_${number}`, MainBlockParams> = {};
|
||||||
range(numMainBlocks, 0, 1).forEach((idx) => {
|
range(numMainBlocks, 0, 1).forEach((idx) => {
|
||||||
middle_flow[`main_block_${idx}`] = extractMainBlockParams(`middle_flow/main_block_${idx}`);
|
middle_flow[`main_block_${idx}`] = extractMainBlockParams(`middle_flow/main_block_${idx}`);
|
||||||
});
|
});
|
||||||
|
|
|
@ -18,7 +18,7 @@ export type TinyXceptionParams = {
|
||||||
reduction_block_0: ReductionBlockParams
|
reduction_block_0: ReductionBlockParams
|
||||||
reduction_block_1: ReductionBlockParams
|
reduction_block_1: ReductionBlockParams
|
||||||
}
|
}
|
||||||
middle_flow: any,
|
middle_flow: Record<`main_block_${number}`, MainBlockParams>,
|
||||||
exit_flow: {
|
exit_flow: {
|
||||||
reduction_block: ReductionBlockParams
|
reduction_block: ReductionBlockParams
|
||||||
separable_conv: SeparableConvParams
|
separable_conv: SeparableConvParams
|
||||||
|
|
|
@ -15,7 +15,7 @@
|
||||||
"experimentalDecorators": true,
|
"experimentalDecorators": true,
|
||||||
"importHelpers": true,
|
"importHelpers": true,
|
||||||
"noFallthroughCasesInSwitch": true,
|
"noFallthroughCasesInSwitch": true,
|
||||||
"noImplicitAny": false,
|
"noImplicitAny": true,
|
||||||
"noImplicitOverride": true,
|
"noImplicitOverride": true,
|
||||||
"noImplicitReturns": true,
|
"noImplicitReturns": true,
|
||||||
"noImplicitThis": true,
|
"noImplicitThis": true,
|
||||||
|
@ -32,7 +32,7 @@
|
||||||
"strictBindCallApply": true,
|
"strictBindCallApply": true,
|
||||||
"strictFunctionTypes": true,
|
"strictFunctionTypes": true,
|
||||||
"strictNullChecks": true,
|
"strictNullChecks": true,
|
||||||
"strictPropertyInitialization": true,
|
"strictPropertyInitialization": true
|
||||||
},
|
},
|
||||||
"formatCodeOptions": { "indentSize": 2, "tabSize": 2 },
|
"formatCodeOptions": { "indentSize": 2, "tabSize": 2 },
|
||||||
"include": ["src"],
|
"include": ["src"],
|
||||||
|
@ -49,6 +49,6 @@
|
||||||
"readme": "none",
|
"readme": "none",
|
||||||
"out": "typedoc",
|
"out": "typedoc",
|
||||||
"logLevel": "Verbose",
|
"logLevel": "Verbose",
|
||||||
"logger": "none",
|
"logger": "none"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue