initial work on skipTime

pull/280/head
Vladimir Mandic 2021-10-22 16:09:52 -04:00
parent 4cec6f5735
commit 5afebaa588
16 changed files with 119 additions and 112 deletions

View File

@ -11,6 +11,9 @@
### **HEAD -> main** 2021/10/22 mandic00@live.com ### **HEAD -> main** 2021/10/22 mandic00@live.com
### **origin/main** 2021/10/22 mandic00@live.com
- add optional autodetected custom wasm path - add optional autodetected custom wasm path
### **2.3.6** 2021/10/21 mandic00@live.com ### **2.3.6** 2021/10/21 mandic00@live.com

View File

@ -137,6 +137,7 @@
"format": "esm", "format": "esm",
"input": "tfjs/tf-browser.ts", "input": "tfjs/tf-browser.ts",
"output": "dist/tfjs.esm.js", "output": "dist/tfjs.esm.js",
"minify": true,
"sourcemap": true, "sourcemap": true,
"external": ["fs", "os", "buffer", "util"] "external": ["fs", "os", "buffer", "util"]
}, },

View File

@ -3,7 +3,7 @@
*/ */
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import type { BodyKeypoint, BodyResult, Box, Point } from '../result'; import type { BodyKeypoint, BodyResult, Box, Point } from '../result';
import type { GraphModel, Tensor } from '../tfjs/types'; import type { GraphModel, Tensor } from '../tfjs/types';
import type { Config } from '../config'; import type { Config } from '../config';
@ -16,6 +16,7 @@ let skipped = Number.MAX_SAFE_INTEGER;
let outputNodes: string[]; // different for lite/full/heavy let outputNodes: string[]; // different for lite/full/heavy
let cache: BodyResult | null = null; let cache: BodyResult | null = null;
let padding: [number, number][] = [[0, 0], [0, 0], [0, 0], [0, 0]]; let padding: [number, number][] = [[0, 0], [0, 0], [0, 0], [0, 0]];
let last = 0;
export async function loadDetect(config: Config): Promise<GraphModel> { export async function loadDetect(config: Config): Promise<GraphModel> {
if (env.initial) models[0] = null; if (env.initial) models[0] = null;
@ -135,10 +136,11 @@ async function detectParts(input: Tensor, config: Config, outputSize: [number, n
export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> { export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0]; const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0];
if ((skipped < (config.body.skipFrames || 0)) && config.skipFrame && cache !== null) { if ((skipped < (config.body.skipFrames || 0)) && ((config.body.skipTime || 0) <= (now() - last)) && config.skipFrame && cache !== null) {
skipped++; skipped++;
} else { } else {
cache = await detectParts(input, config, outputSize); cache = await detectParts(input, config, outputSize);
last = now();
skipped = 0; skipped = 0;
} }
if (cache) return [cache]; if (cache) return [cache];

View File

@ -4,7 +4,7 @@
* Based on: [**EfficientPose**](https://github.com/daniegr/EfficientPose) * Based on: [**EfficientPose**](https://github.com/daniegr/EfficientPose)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as coords from './efficientposecoords'; import * as coords from './efficientposecoords';
import type { BodyResult, Point } from '../result'; import type { BodyResult, Point } from '../result';
@ -13,7 +13,7 @@ import type { Config } from '../config';
import { env } from '../util/env'; import { env } from '../util/env';
let model: GraphModel | null; let model: GraphModel | null;
let last = 0;
const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} }; const cache: BodyResult = { id: 0, keypoints: [], box: [0, 0, 0, 0], boxRaw: [0, 0, 0, 0], score: 0, annotations: {} };
// const keypoints: Array<BodyKeypoint> = []; // const keypoints: Array<BodyKeypoint> = [];
@ -50,12 +50,7 @@ function max2d(inputs, minScore) {
} }
export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> { export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
/** blazepose caching if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0 && ((config.body.skipTime || 0) <= (now() - last))) {
* not fully implemented
* 1. if skipFrame returned cached
* 2. run detection based on squared full frame
*/
if ((skipped < (config.body?.skipFrames || 0)) && config.skipFrame && Object.keys(cache.keypoints).length > 0) {
skipped++; skipped++;
return [cache]; return [cache];
} }
@ -71,6 +66,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
let resT; let resT;
if (config.body.enabled) resT = await model?.predict(tensor); if (config.body.enabled) resT = await model?.predict(tensor);
last = now();
tf.dispose(tensor); tf.dispose(tensor);
if (resT) { if (resT) {

View File

@ -4,7 +4,7 @@
* Based on: [**MoveNet**](https://blog.tensorflow.org/2021/05/next-generation-pose-detection-with-movenet-and-tensorflowjs.html) * Based on: [**MoveNet**](https://blog.tensorflow.org/2021/05/next-generation-pose-detection-with-movenet-and-tensorflowjs.html)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as box from '../util/box'; import * as box from '../util/box';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as coords from './movenetcoords'; import * as coords from './movenetcoords';
@ -23,9 +23,11 @@ let skipped = Number.MAX_SAFE_INTEGER;
const cache: { const cache: {
boxes: Array<Box>, // unused boxes: Array<Box>, // unused
bodies: Array<BodyResult>; bodies: Array<BodyResult>;
last: number,
} = { } = {
boxes: [], boxes: [],
bodies: [], bodies: [],
last: 0,
}; };
export async function load(config: Config): Promise<GraphModel> { export async function load(config: Config): Promise<GraphModel> {
@ -129,17 +131,10 @@ async function parseMultiPose(res, config, image, inputBox) {
} }
export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> { export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
/** movenet caching
* 1. if skipFrame returned cached
* 2. if enough cached boxes run using cached boxes
* 3. if not enough detected bodies rerun using full frame
* 4. regenerate cached boxes based on current keypoints
*/
if (!model || !model?.inputs[0].shape) return []; // something is wrong with the model if (!model || !model?.inputs[0].shape) return []; // something is wrong with the model
if (!config.skipFrame) cache.boxes.length = 0; // allowed to use cache or not if (!config.skipFrame) cache.boxes.length = 0; // allowed to use cache or not
skipped++; // increment skip frames skipped++; // increment skip frames
if (config.skipFrame && (skipped <= (config.body.skipFrames || 0))) { if (config.skipFrame && (skipped <= (config.body.skipFrames || 0) && ((config.body.skipTime || 0) <= (now() - cache.last)))) {
return cache.bodies; // return cached results without running anything return cache.bodies; // return cached results without running anything
} }
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
@ -181,6 +176,7 @@ export async function predict(input: Tensor, config: Config): Promise<BodyResult
// run detection on squared input and no cached boxes // run detection on squared input and no cached boxes
t.input = fix.padInput(input, inputSize); t.input = fix.padInput(input, inputSize);
t.res = await model?.predict(t.input) as Tensor; t.res = await model?.predict(t.input) as Tensor;
cache.last = now();
const res = await t.res.array(); const res = await t.res.array();
cache.bodies = (t.res.shape[2] === 17) cache.bodies = (t.res.shape[2] === 17)
? await parseSinglePose(res, config, input, [0, 0, 1, 1]) ? await parseSinglePose(res, config, input, [0, 0, 1, 1])

View File

@ -1,53 +1,42 @@
/* eslint-disable indent */ /* eslint-disable indent */
/* eslint-disable no-multi-spaces */ /* eslint-disable no-multi-spaces */
/** Dectector part of face configuration */ export interface GenericConfig {
export interface FaceDetectorConfig { enabled: boolean,
modelPath: string, modelPath: string,
skipFrames: number,
skipTime: number,
}
/** Dectector part of face configuration */
export interface FaceDetectorConfig extends GenericConfig {
rotation: boolean, rotation: boolean,
maxDetected: number, maxDetected: number,
skipFrames: number,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
return: boolean, return: boolean,
} }
/** Mesh part of face configuration */ /** Mesh part of face configuration */
export interface FaceMeshConfig { export type FaceMeshConfig = GenericConfig
enabled: boolean,
modelPath: string,
}
/** Iris part of face configuration */ /** Iris part of face configuration */
export interface FaceIrisConfig { export type FaceIrisConfig = GenericConfig
enabled: boolean,
modelPath: string,
}
/** Description or face embedding part of face configuration /** Description or face embedding part of face configuration
* - also used by age and gender detection * - also used by age and gender detection
*/ */
export interface FaceDescriptionConfig { export interface FaceDescriptionConfig extends GenericConfig {
enabled: boolean,
modelPath: string,
skipFrames: number,
minConfidence: number, minConfidence: number,
} }
/** Emotion part of face configuration */ /** Emotion part of face configuration */
export interface FaceEmotionConfig { export interface FaceEmotionConfig extends GenericConfig {
enabled: boolean,
minConfidence: number, minConfidence: number,
skipFrames: number,
modelPath: string,
} }
/** Emotion part of face configuration */ /** Emotion part of face configuration */
export interface FaceAntiSpoofConfig { export type FaceAntiSpoofConfig = GenericConfig
enabled: boolean,
skipFrames: number,
modelPath: string,
}
/** Controlls and configures all face-specific options: /** Controlls and configures all face-specific options:
* - face detection, face mesh detection, age, gender, emotion detection and face description * - face detection, face mesh detection, age, gender, emotion detection and face description
@ -86,18 +75,15 @@ export interface FaceConfig {
* Changing `modelPath` will change module responsible for hand detection and tracking * Changing `modelPath` will change module responsible for hand detection and tracking
* Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json` * Allowed values are `posenet.json`, `blazepose.json`, `efficientpose.json`, `movenet-lightning.json`, `movenet-thunder.json`, `movenet-multipose.json`
*/ */
export interface BodyConfig { export interface BodyConfig extends GenericConfig {
enabled: boolean,
modelPath: string,
maxDetected: number, maxDetected: number,
minConfidence: number, minConfidence: number,
skipFrames: number,
detector?: { detector?: {
modelPath: string modelPath: string
}, },
} }
/** Controlls and configures all hand detection specific options /** Controls and configures all hand detection specific options
* *
* Parameters: * Parameters:
* - enabled: true/false * - enabled: true/false
@ -113,10 +99,8 @@ export interface BodyConfig {
* Changing `detector.modelPath` will change module responsible for hand detection and tracking * Changing `detector.modelPath` will change module responsible for hand detection and tracking
* Allowed values are `handdetect.json` and `handtrack.json` * Allowed values are `handdetect.json` and `handtrack.json`
*/ */
export interface HandConfig { export interface HandConfig extends GenericConfig {
enabled: boolean,
rotation: boolean, rotation: boolean,
skipFrames: number,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
maxDetected: number, maxDetected: number,
@ -139,13 +123,10 @@ export interface HandConfig {
* Changing `modelPath` will change module responsible for hand detection and tracking * Changing `modelPath` will change module responsible for hand detection and tracking
* Allowed values are `mb3-centernet.json` and `nanodet.json` * Allowed values are `mb3-centernet.json` and `nanodet.json`
*/ */
export interface ObjectConfig { export interface ObjectConfig extends GenericConfig {
enabled: boolean,
modelPath: string,
minConfidence: number, minConfidence: number,
iouThreshold: number, iouThreshold: number,
maxDetected: number, maxDetected: number,
skipFrames: number,
} }
/** Controlls and configures all body segmentation module /** Controlls and configures all body segmentation module
@ -368,9 +349,8 @@ const config: Config = {
// should be set to the minimum number for performance // should be set to the minimum number for performance
skipFrames: 11, // how many max frames to go without re-running the face bounding box detector skipFrames: 11, // how many max frames to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
// e.g., if model is running st 25 FPS, we can re-use existing bounding skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// box for updated face analysis as the head does not move fast // only used when cacheSensitivity is not zero
// in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.2, // threshold for discarding a prediction minConfidence: 0.2, // threshold for discarding a prediction
iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.1, // ammount of overlap between two detected objects before one object is removed
return: false, // return extracted face as tensor return: false, // return extracted face as tensor
@ -393,6 +373,8 @@ const config: Config = {
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
skipFrames: 12, // how max many frames to go without re-running the detector skipFrames: 12, // how max many frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero
modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath modelPath: 'emotion.json', // face emotion model, can be absolute path or relative to modelBasePath
}, },
@ -403,6 +385,8 @@ const config: Config = {
// can be either absolute path or relative to modelBasePath // can be either absolute path or relative to modelBasePath
skipFrames: 13, // how many max frames to go without re-running the detector skipFrames: 13, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero
minConfidence: 0.1, // threshold for discarding a prediction minConfidence: 0.1, // threshold for discarding a prediction
}, },
@ -410,6 +394,8 @@ const config: Config = {
enabled: false, enabled: false,
skipFrames: 14, // how max many frames to go without re-running the detector skipFrames: 14, // how max many frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero
modelPath: 'antispoof.json', // face description model modelPath: 'antispoof.json', // face description model
// can be either absolute path or relative to modelBasePath // can be either absolute path or relative to modelBasePath
}, },
@ -429,6 +415,8 @@ const config: Config = {
minConfidence: 0.3, // threshold for discarding a prediction minConfidence: 0.3, // threshold for discarding a prediction
skipFrames: 1, // how many max frames to go without re-running the detector skipFrames: 1, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero
}, },
hand: { hand: {
@ -438,9 +426,8 @@ const config: Config = {
// only valid for `handdetect` variation // only valid for `handdetect` variation
skipFrames: 2, // how many max frames to go without re-running the hand bounding box detector skipFrames: 2, // how many max frames to go without re-running the hand bounding box detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
// e.g., if model is running st 25 FPS, we can re-use existing bounding skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// box for updated hand skeleton analysis as the hand // only used when cacheSensitivity is not zero
// hasn't moved much in short time (10 * 1/25 = 0.25 sec)
minConfidence: 0.50, // threshold for discarding a prediction minConfidence: 0.50, // threshold for discarding a prediction
iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed iouThreshold: 0.2, // ammount of overlap between two detected objects before one object is removed
maxDetected: -1, // maximum number of hands detected in the input maxDetected: -1, // maximum number of hands detected in the input
@ -465,6 +452,8 @@ const config: Config = {
maxDetected: 10, // maximum number of objects detected in the input maxDetected: 10, // maximum number of objects detected in the input
skipFrames: 15, // how many max frames to go without re-running the detector skipFrames: 15, // how many max frames to go without re-running the detector
// only used when cacheSensitivity is not zero // only used when cacheSensitivity is not zero
skipTime: 2000, // how many ms to go without re-running the face bounding box detector
// only used when cacheSensitivity is not zero
}, },
segmentation: { segmentation: {

View File

@ -2,7 +2,7 @@
* Anti-spoofing model implementation * Anti-spoofing model implementation
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import type { Config } from '../config'; import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types'; import type { GraphModel, Tensor } from '../tfjs/types';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
@ -12,6 +12,7 @@ let model: GraphModel | null;
const cached: Array<number> = []; const cached: Array<number> = [];
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
let lastCount = 0; let lastCount = 0;
let last = 0;
export async function load(config: Config): Promise<GraphModel> { export async function load(config: Config): Promise<GraphModel> {
if (env.initial) model = null; if (env.initial) model = null;
@ -25,7 +26,7 @@ export async function load(config: Config): Promise<GraphModel> {
export async function predict(image: Tensor, config: Config, idx, count) { export async function predict(image: Tensor, config: Config, idx, count) {
if (!model) return null; if (!model) return null;
if ((skipped < (config.face.antispoof?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && cached[idx]) { if ((skipped < (config.face.antispoof?.skipFrames || 0)) && ((config.face.antispoof?.skipTime || 0) <= (now() - last)) && config.skipFrame && (lastCount === count) && cached[idx]) {
skipped++; skipped++;
return cached[idx]; return cached[idx];
} }
@ -36,6 +37,7 @@ export async function predict(image: Tensor, config: Config, idx, count) {
const num = (await res.data())[0]; const num = (await res.data())[0];
cached[idx] = Math.round(100 * num) / 100; cached[idx] = Math.round(100 * num) / 100;
lastCount = count; lastCount = count;
last = now();
tf.dispose([resize, res]); tf.dispose([resize, res]);
resolve(cached[idx]); resolve(cached[idx]);
}); });

View File

@ -7,7 +7,7 @@
* - Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view) * - Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import * as blazeface from './blazeface'; import * as blazeface from './blazeface';
import * as util from './facemeshutil'; import * as util from './facemeshutil';
@ -23,11 +23,14 @@ let boxCache: Array<BoxCache> = [];
let model: GraphModel | null = null; let model: GraphModel | null = null;
let inputSize = 0; let inputSize = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
let lastTime = 0;
let detectedFaces = 0; let detectedFaces = 0;
export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> { export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0))) { // reset cached boxes // reset cached boxes
if (!config.skipFrame || (((detectedFaces !== config.face.detector?.maxDetected) || !config.face.mesh?.enabled)) && (skipped > (config.face.detector?.skipFrames || 0) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)))) {
const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector const newBoxes = await blazeface.getBoxes(input, config); // get results from blazeface detector
lastTime = now();
boxCache = []; // empty cache boxCache = []; // empty cache
for (const possible of newBoxes.boxes) { // extract data from detector for (const possible of newBoxes.boxes) { // extract data from detector
const startPoint = await possible.box.startPoint.data() as unknown as Point; const startPoint = await possible.box.startPoint.data() as unknown as Point;

View File

@ -7,7 +7,7 @@
* Based on: [**HSE-FaceRes**](https://github.com/HSE-asavchenko/HSE_FaceRec_tf) * Based on: [**HSE-FaceRes**](https://github.com/HSE-asavchenko/HSE_FaceRec_tf)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import type { Tensor, GraphModel } from '../tfjs/types'; import type { Tensor, GraphModel } from '../tfjs/types';
import type { Config } from '../config'; import type { Config } from '../config';
@ -21,6 +21,7 @@ const last: Array<{
descriptor: number[], descriptor: number[],
}> = []; }> = [];
let lastTime = 0;
let lastCount = 0; let lastCount = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
@ -90,15 +91,12 @@ export function enhance(input): Tensor {
export async function predict(image: Tensor, config: Config, idx, count) { export async function predict(image: Tensor, config: Config, idx, count) {
if (!model) return null; if (!model) return null;
if ((skipped < (config.face.description?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) { if ((skipped < (config.face.description?.skipFrames || 0)) && ((config.face.description?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx]?.age && (last[idx]?.age > 0)) {
skipped++; skipped++;
return last[idx]; return last[idx];
} }
skipped = 0; skipped = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const enhanced = enhance(image);
let resT;
const obj = { const obj = {
age: <number>0, age: <number>0,
gender: <string>'unknown', gender: <string>'unknown',
@ -106,11 +104,13 @@ export async function predict(image: Tensor, config: Config, idx, count) {
descriptor: <number[]>[], descriptor: <number[]>[],
}; };
if (config.face.description?.enabled) resT = await model?.predict(enhanced); if (config.face.description?.enabled) {
tf.dispose(enhanced); const enhanced = enhance(image);
const resT = await model?.predict(enhanced) as Tensor[];
if (resT) { lastTime = now();
const gender = await resT.find((t) => t.shape[1] === 1).data(); tf.dispose(enhanced);
const genderT = await resT.find((t) => t.shape[1] === 1) as Tensor;
const gender = await genderT.data();
const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100; const confidence = Math.trunc(200 * Math.abs((gender[0] - 0.5))) / 100;
if (confidence > (config.face.description?.minConfidence || 0)) { if (confidence > (config.face.description?.minConfidence || 0)) {
obj.gender = gender[0] <= 0.5 ? 'female' : 'male'; obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
@ -119,15 +119,16 @@ export async function predict(image: Tensor, config: Config, idx, count) {
const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1); const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1);
const age = (await argmax.data())[0]; const age = (await argmax.data())[0];
tf.dispose(argmax); tf.dispose(argmax);
const all = await resT.find((t) => t.shape[1] === 100).data(); const ageT = resT.find((t) => t.shape[1] === 100) as Tensor;
const all = await ageT.data();
obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10; obj.age = Math.round(all[age - 1] > all[age + 1] ? 10 * age - 100 * all[age - 1] : 10 * age + 100 * all[age + 1]) / 10;
const desc = resT.find((t) => t.shape[1] === 1024); const desc = resT.find((t) => t.shape[1] === 1024);
// const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8 // const reshape = desc.reshape([128, 8]); // reshape large 1024-element descriptor to 128 x 8
// const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor // const reduce = reshape.logSumExp(1); // reduce 2nd dimension by calculating logSumExp on it which leaves us with 128-element descriptor
const descriptor = desc ? await desc.data() : <number[]>[];
const descriptor = await desc.data(); // obj.descriptor = [...descriptor];
obj.descriptor = [...descriptor]; obj.descriptor = Array.from(descriptor);
resT.forEach((t) => tf.dispose(t)); resT.forEach((t) => tf.dispose(t));
} }
last[idx] = obj; last[idx] = obj;

View File

@ -4,7 +4,7 @@
* [**Oarriaga**](https://github.com/oarriaga/face_classification) * [**Oarriaga**](https://github.com/oarriaga/face_classification)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import type { Config } from '../config'; import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types'; import type { GraphModel, Tensor } from '../tfjs/types';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
@ -15,6 +15,7 @@ let model: GraphModel | null;
// let last: Array<{ score: number, emotion: string }> = []; // let last: Array<{ score: number, emotion: string }> = [];
const last: Array<Array<{ score: number, emotion: string }>> = []; const last: Array<Array<{ score: number, emotion: string }>> = [];
let lastCount = 0; let lastCount = 0;
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
// tuning values // tuning values
@ -32,39 +33,40 @@ export async function load(config: Config): Promise<GraphModel> {
export async function predict(image: Tensor, config: Config, idx, count) { export async function predict(image: Tensor, config: Config, idx, count) {
if (!model) return null; if (!model) return null;
if ((skipped < (config.face.emotion?.skipFrames || 0)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) { if ((skipped < (config.face.emotion?.skipFrames || 0)) && ((config.face.emotion?.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (lastCount === count) && last[idx] && (last[idx].length > 0)) {
skipped++; skipped++;
return last[idx]; return last[idx];
} }
skipped = 0; skipped = 0;
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false);
const [red, green, blue] = tf.split(resize, 3, 3);
tf.dispose(resize);
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]);
tf.dispose(red);
tf.dispose(green);
tf.dispose(blue);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
tf.dispose(redNorm);
tf.dispose(greenNorm);
tf.dispose(blueNorm);
const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
tf.dispose(grayscale);
const obj: Array<{ score: number, emotion: string }> = []; const obj: Array<{ score: number, emotion: string }> = [];
if (config.face.emotion?.enabled) { if (config.face.emotion?.enabled) {
const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape ? model.inputs[0].shape[2] : 0, model?.inputs[0].shape ? model.inputs[0].shape[1] : 0], false);
const [red, green, blue] = tf.split(resize, 3, 3);
tf.dispose(resize);
// weighted rgb to grayscale: https://www.mathworks.com/help/matlab/ref/rgb2gray.html
const redNorm = tf.mul(red, rgb[0]);
const greenNorm = tf.mul(green, rgb[1]);
const blueNorm = tf.mul(blue, rgb[2]);
tf.dispose(red);
tf.dispose(green);
tf.dispose(blue);
const grayscale = tf.addN([redNorm, greenNorm, blueNorm]);
tf.dispose(redNorm);
tf.dispose(greenNorm);
tf.dispose(blueNorm);
const normalize = tf.tidy(() => tf.mul(tf.sub(grayscale, 0.5), 2));
tf.dispose(grayscale);
const emotionT = await model?.predict(normalize) as Tensor; // result is already in range 0..1, no need for additional activation const emotionT = await model?.predict(normalize) as Tensor; // result is already in range 0..1, no need for additional activation
lastTime = now();
const data = await emotionT.data(); const data = await emotionT.data();
tf.dispose(emotionT); tf.dispose(emotionT);
for (let i = 0; i < data.length; i++) { for (let i = 0; i < data.length; i++) {
if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] }); if (data[i] > (config.face.emotion?.minConfidence || 0)) obj.push({ score: Math.min(0.99, Math.trunc(100 * data[i]) / 100), emotion: annotations[i] });
} }
obj.sort((a, b) => b.score - a.score); obj.sort((a, b) => b.score - a.score);
tf.dispose(normalize);
} }
tf.dispose(normalize);
last[idx] = obj; last[idx] = obj;
lastCount = count; lastCount = count;
resolve(obj); resolve(obj);

View File

@ -6,15 +6,15 @@
* Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis * Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import type { Config } from '../config'; import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types'; import type { GraphModel, Tensor } from '../tfjs/types';
import { env } from '../util/env'; import { env } from '../util/env';
let model: GraphModel | null; let model: GraphModel | null;
let last = { age: 0 }; let last = { age: 0 };
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -33,7 +33,7 @@ export async function load(config: Config | any) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function predict(image: Tensor, config: Config | any) { export async function predict(image: Tensor, config: Config | any) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.age.skipFrames) && config.skipFrame && last.age && (last.age > 0)) { if ((skipped < config.face.age.skipFrames) && ((config.face.age.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.age && (last.age > 0)) {
skipped++; skipped++;
return last; return last;
} }
@ -48,6 +48,7 @@ export async function predict(image: Tensor, config: Config | any) {
const obj = { age: 0 }; const obj = { age: 0 };
if (config.face.age.enabled) ageT = await model.predict(enhance); if (config.face.age.enabled) ageT = await model.predict(enhance);
lastTime = now();
tf.dispose(enhance); tf.dispose(enhance);
if (ageT) { if (ageT) {

View File

@ -6,7 +6,7 @@
* Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis * Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import type { Config } from '../config'; import type { Config } from '../config';
import type { GraphModel, Tensor } from '../tfjs/types'; import type { GraphModel, Tensor } from '../tfjs/types';
@ -14,6 +14,7 @@ import { env } from '../util/env';
let model: GraphModel | null; let model: GraphModel | null;
let last = { gender: '' }; let last = { gender: '' };
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
let alternative = false; let alternative = false;
@ -35,7 +36,7 @@ export async function load(config: Config | any) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function predict(image: Tensor, config: Config | any) { export async function predict(image: Tensor, config: Config | any) {
if (!model) return null; if (!model) return null;
if ((skipped < config.face.gender.skipFrames) && config.skipFrame && last.gender !== '') { if ((skipped < config.face.gender.skipFrames) && ((config.face.gender.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && last.gender !== '') {
skipped++; skipped++;
return last; return last;
} }
@ -63,6 +64,7 @@ export async function predict(image: Tensor, config: Config | any) {
const obj = { gender: '', confidence: 0 }; const obj = { gender: '', confidence: 0 };
if (config.face.gender.enabled) genderT = await model.predict(enhance); if (config.face.gender.enabled) genderT = await model.predict(enhance);
lastTime = now();
tf.dispose(enhance); tf.dispose(enhance);
if (genderT) { if (genderT) {

View File

@ -8,12 +8,14 @@ import * as util from './handposeutil';
import type * as detector from './handposedetector'; import type * as detector from './handposedetector';
import type { Tensor, GraphModel } from '../tfjs/types'; import type { Tensor, GraphModel } from '../tfjs/types';
import { env } from '../util/env'; import { env } from '../util/env';
import { now } from '../util/util';
const palmBoxEnlargeFactor = 5; // default 3 const palmBoxEnlargeFactor = 5; // default 3
const handBoxEnlargeFactor = 1.65; // default 1.65 const handBoxEnlargeFactor = 1.65; // default 1.65
const palmLandmarkIds = [0, 5, 9, 13, 17, 1, 2]; const palmLandmarkIds = [0, 5, 9, 13, 17, 1, 2];
const palmLandmarksPalmBase = 0; const palmLandmarksPalmBase = 0;
const palmLandmarksMiddleFingerBase = 2; const palmLandmarksMiddleFingerBase = 2;
let lastTime = 0;
export class HandPipeline { export class HandPipeline {
handDetector: detector.HandDetector; handDetector: detector.HandDetector;
@ -90,7 +92,7 @@ export class HandPipeline {
let boxes; let boxes;
// console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector? // console.log('handpipeline:estimateHands:skip criteria', this.skipped, config.hand.skipFrames, !config.hand.landmarks, !config.skipFrame); // should skip hand detector?
if ((this.skipped === 0) || (this.skipped > config.hand.skipFrames) || !config.hand.landmarks || !config.skipFrame) { if ((this.skipped === 0) || ((this.skipped > config.hand.skipFrames) && ((config.hand.skipTime || 0) <= (now() - lastTime))) || !config.hand.landmarks || !config.skipFrame) {
boxes = await this.handDetector.estimateHandBounds(image, config); boxes = await this.handDetector.estimateHandBounds(image, config);
this.skipped = 0; this.skipped = 0;
} }
@ -121,6 +123,7 @@ export class HandPipeline {
tf.dispose(croppedInput); tf.dispose(croppedInput);
tf.dispose(rotatedImage); tf.dispose(rotatedImage);
const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>; const [confidenceT, keypoints] = await this.handPoseModel.predict(handImage) as Array<Tensor>;
lastTime = now();
tf.dispose(handImage); tf.dispose(handImage);
const confidence = (await confidenceT.data())[0]; const confidence = (await confidenceT.data())[0];
tf.dispose(confidenceT); tf.dispose(confidenceT);

View File

@ -6,7 +6,7 @@
* - Hand Tracking: [**HandTracking**](https://github.com/victordibia/handtracking) * - Hand Tracking: [**HandTracking**](https://github.com/victordibia/handtracking)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as box from '../util/box'; import * as box from '../util/box';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import type { HandResult, Box, Point } from '../result'; import type { HandResult, Box, Point } from '../result';
@ -29,6 +29,7 @@ const maxDetectorResolution = 512;
const detectorExpandFact = 1.4; const detectorExpandFact = 1.4;
let skipped = 0; let skipped = 0;
let lastTime = 0;
let outputSize: [number, number] = [0, 0]; let outputSize: [number, number] = [0, 0];
type HandDetectResult = { type HandDetectResult = {
@ -184,7 +185,7 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
export async function predict(input: Tensor, config: Config): Promise<HandResult[]> { export async function predict(input: Tensor, config: Config): Promise<HandResult[]> {
/** handtrack caching /** handtrack caching
* 1. if skipFrame returned cached * 1. if skipFrame returned cached
* 2. if any cached results but although not sure if its enough we continute anyhow for 5x skipframes * 2. if any cached results but although not sure if its enough we continute anyhow for 3x skipframes
* 3. if not skipframe or eventually rerun detector to generated new cached boxes and reset skipped * 3. if not skipframe or eventually rerun detector to generated new cached boxes and reset skipped
* 4. generate cached boxes based on detected keypoints * 4. generate cached boxes based on detected keypoints
*/ */
@ -192,16 +193,17 @@ export async function predict(input: Tensor, config: Config): Promise<HandResult
outputSize = [input.shape[2] || 0, input.shape[1] || 0]; outputSize = [input.shape[2] || 0, input.shape[1] || 0];
skipped++; // increment skip frames skipped++; // increment skip frames
if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0))) { if (config.skipFrame && (skipped <= (config.hand.skipFrames || 0)) && ((config.hand.skipTime || 0) <= (now() - lastTime))) {
return cache.hands; // return cached results without running anything return cache.hands; // return cached results without running anything
} }
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
if (config.skipFrame && cache.hands.length === config.hand.maxDetected) { // we have all detected hands if (config.skipFrame && cache.hands.length === config.hand.maxDetected) { // we have all detected hands
cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config))); cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
} else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && cache.hands.length > 0) { // we have some cached results but although not sure if its enough we continute anyhow for bit longer } else if (config.skipFrame && skipped < 3 * (config.hand.skipFrames || 0) && ((config.hand.skipTime || 0) <= 3 * (now() - lastTime)) && cache.hands.length > 0) { // we have some cached results: maybe not enough but anyhow continue for bit longer
cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config))); cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
} else { // finally rerun detector } else { // finally rerun detector
cache.boxes = await detectHands(input, config); cache.boxes = await detectHands(input, config);
lastTime = now();
cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config))); cache.hands = await Promise.all(cache.boxes.map((handBox) => detectFingers(input, handBox, config)));
skipped = 0; skipped = 0;
} }

View File

@ -4,7 +4,7 @@
* Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet) * Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import { labels } from './labels'; import { labels } from './labels';
import type { ObjectResult, Box } from '../result'; import type { ObjectResult, Box } from '../result';
@ -16,6 +16,7 @@ import { fakeOps } from '../tfjs/backend';
let model: GraphModel | null; let model: GraphModel | null;
let inputSize = 0; let inputSize = 0;
let last: ObjectResult[] = []; let last: ObjectResult[] = [];
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
export async function load(config: Config): Promise<GraphModel> { export async function load(config: Config): Promise<GraphModel> {
@ -78,7 +79,7 @@ async function process(res: Tensor | null, outputShape, config: Config) {
} }
export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> { export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> {
if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) { if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
skipped++; skipped++;
return last; return last;
} }
@ -88,6 +89,7 @@ export async function predict(input: Tensor, config: Config): Promise<ObjectResu
const outputSize = [input.shape[2], input.shape[1]]; const outputSize = [input.shape[2], input.shape[1]];
const resize = tf.image.resizeBilinear(input, [inputSize, inputSize]); const resize = tf.image.resizeBilinear(input, [inputSize, inputSize]);
const objectT = config.object.enabled ? model?.execute(resize, ['tower_0/detections']) as Tensor : null; const objectT = config.object.enabled ? model?.execute(resize, ['tower_0/detections']) as Tensor : null;
lastTime = now();
tf.dispose(resize); tf.dispose(resize);
const obj = await process(objectT, outputSize, config); const obj = await process(objectT, outputSize, config);

View File

@ -4,7 +4,7 @@
* Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet) * Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet)
*/ */
import { log, join } from '../util/util'; import { log, join, now } from '../util/util';
import * as tf from '../../dist/tfjs.esm.js'; import * as tf from '../../dist/tfjs.esm.js';
import { labels } from './labels'; import { labels } from './labels';
import type { ObjectResult, Box } from '../result'; import type { ObjectResult, Box } from '../result';
@ -14,6 +14,7 @@ import { env } from '../util/env';
let model; let model;
let last: Array<ObjectResult> = []; let last: Array<ObjectResult> = [];
let lastTime = 0;
let skipped = Number.MAX_SAFE_INTEGER; let skipped = Number.MAX_SAFE_INTEGER;
const scaleBox = 2.5; // increase box size const scaleBox = 2.5; // increase box size
@ -106,7 +107,7 @@ async function process(res, inputSize, outputShape, config) {
} }
export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> { export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> {
if ((skipped < (config.object.skipFrames || 0)) && config.skipFrame && (last.length > 0)) { if ((skipped < (config.object.skipFrames || 0)) && ((config.object.skipTime || 0) <= (now() - lastTime)) && config.skipFrame && (last.length > 0)) {
skipped++; skipped++;
return last; return last;
} }
@ -122,6 +123,7 @@ export async function predict(image: Tensor, config: Config): Promise<ObjectResu
let objectT; let objectT;
if (config.object.enabled) objectT = await model.predict(transpose); if (config.object.enabled) objectT = await model.predict(transpose);
lastTime = now();
tf.dispose(transpose); tf.dispose(transpose);
const obj = await process(objectT, model.inputSize, outputSize, config); const obj = await process(objectT, model.inputSize, outputSize, config);