mirror of https://github.com/vladmandic/human
tensor rank strong typechecks
Vladimir Mandic
parent
41aeadf00f
commit
7a82a73273
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
|
||||
"mainEntryPointFilePath": "types/lib/src/human.d.ts",
|
||||
"bundledPackages": ["@tensorflow/tfjs-core", "@tensorflow/tfjs-converter", "@tensorflow/tfjs-data", "@tensorflow/tfjs-layers"],
|
||||
"bundledPackages": ["@tensorflow/tfjs-core", "@tensorflow/tfjs-converter"],
|
||||
"compiler": {
|
||||
"skipLibCheck": false
|
||||
},
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
"clean": ["clean"]
|
||||
},
|
||||
"clean": {
|
||||
"locations": ["dist/*", "types/lib/*", "typedoc/*"]
|
||||
"locations": ["dist/*", "types/*", "typedoc/*"]
|
||||
},
|
||||
"lint": {
|
||||
"locations": [ "*.json", "src/**/*.ts", "test/**/*.js", "demo/**/*.js" ],
|
||||
|
|
@ -24,8 +24,8 @@
|
|||
"serve": {
|
||||
"sslKey": "node_modules/@vladmandic/build/cert/https.key",
|
||||
"sslCrt": "node_modules/@vladmandic/build/cert/https.crt",
|
||||
"httpPort": 10030,
|
||||
"httpsPort": 10031,
|
||||
"httpPort": 8000,
|
||||
"httpsPort": 8001,
|
||||
"documentRoot": ".",
|
||||
"defaultFolder": "demo",
|
||||
"defaultFile": "index.html"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# @vladmandic/human
|
||||
|
||||
Version: **2.11.1**
|
||||
Version: **3.0.0**
|
||||
Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition**
|
||||
|
||||
Author: **Vladimir Mandic <mandic00@live.com>**
|
||||
|
|
@ -9,7 +9,10 @@
|
|||
|
||||
## Changelog
|
||||
|
||||
### **HEAD -> main** 2022/10/09 mandic00@live.com
|
||||
### **HEAD -> main** 2022/10/13 mandic00@live.com
|
||||
|
||||
|
||||
### **release: 2.11.1** 2022/10/09 mandic00@live.com
|
||||
|
||||
|
||||
### **2.11.1** 2022/10/09 mandic00@live.com
|
||||
|
|
|
|||
32
README.md
32
README.md
|
|
@ -12,7 +12,8 @@
|
|||
**Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis,**
|
||||
**Age & Gender & Emotion Prediction, Gaze Tracking, Gesture Recognition, Body Segmentation**
|
||||
|
||||
JavaScript module using TensorFlow/JS Machine Learning library
|
||||
<br>
|
||||
|
||||
## Highlights
|
||||
|
||||
- Compatible with most server-side and client-side environments and frameworks
|
||||
|
|
@ -239,7 +240,7 @@ Additionally, `HTMLVideoElement`, `HTMLMediaElement` can be a standard `<video>`
|
|||
e.g.: **HLS** (*HTTP Live Streaming*) using `hls.js` or **DASH** (*Dynamic Adaptive Streaming over HTTP*) using `dash.js`
|
||||
- **WebRTC** media track using built-in support
|
||||
|
||||
<br>
|
||||
<br><hr><br>
|
||||
|
||||
## Code Examples
|
||||
|
||||
|
|
@ -375,6 +376,27 @@ drawResults(); // start draw loop
|
|||
|
||||
And for even better results, you can run detection in a separate web worker thread
|
||||
|
||||
<br><hr><br>
|
||||
|
||||
## TypeDefs
|
||||
|
||||
`Human` is written using TypeScript strong typing and ships with full **TypeDefs** for all classes defined by the library bundled in `types/human.d.ts` and enabled by default
|
||||
|
||||
*Note*: This does not include embedded `tfjs`
|
||||
If you want to use embedded `tfjs` inside `Human` (`human.tf` namespace) and still full **typedefs**, add this code:
|
||||
|
||||
```js
|
||||
import type * as tfjs from '@vladmandic/human/dist/tfjs.esm';
|
||||
...
|
||||
const tf = human.tf as typeof tfjs;
|
||||
```
|
||||
|
||||
This is not enabled by default as `Human` does not ship with full **TFJS TypeDefs** due to size considerations
|
||||
Enabling `tfjs` TypeDefs as above creates additional project (dev-only as only types are required) dependencies as defined in `@vladmandic/human/dist/tfjs.esm.d.ts`:
|
||||
|
||||
@tensorflow/tfjs-core, @tensorflow/tfjs-converter, @tensorflow/tfjs-backend-wasm, @tensorflow/tfjs-backend-webgl
|
||||
|
||||
|
||||
<br><hr><br>
|
||||
|
||||
## Default models
|
||||
|
|
@ -404,9 +426,9 @@ For more info, see [**Configuration Details**](https://github.com/vladmandic/hum
|
|||
|
||||
<br><hr><br>
|
||||
|
||||
`Human` library is written in `TypeScript` [4.8](https://www.typescriptlang.org/docs/handbook/intro.html)
|
||||
Conforming to latest `JavaScript` [ECMAScript version 2022](https://262.ecma-international.org/) standard
|
||||
Build target is `JavaScript` [EMCAScript version 2018](https://262.ecma-international.org/11.0/)
|
||||
`Human` library is written in [TypeScript](https://www.typescriptlang.org/docs/handbook/intro.html) **4.8** using [TensorFlow/JS](https://www.tensorflow.org/js/) **4.0** and conforming to latest `JavaScript` [ECMAScript version 2022](https://262.ecma-international.org/) standard
|
||||
|
||||
Build target for distributables is `JavaScript` [EMCAScript version 2018](https://262.ecma-international.org/9.0/)
|
||||
|
||||
<br>
|
||||
|
||||
|
|
|
|||
2
TODO.md
2
TODO.md
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
## Work-in-Progress
|
||||
|
||||
- `src/tfjs/tfjs.esm.d.ts`
|
||||
- `src/tfjs/long.d.ts`
|
||||
|
||||
<hr><br>
|
||||
|
||||
|
|
|
|||
4
build.js
4
build.js
|
|
@ -93,8 +93,8 @@ async function main() {
|
|||
const build = new Build();
|
||||
await build.run('production');
|
||||
// patch tfjs typedefs
|
||||
log.state('Copy:', { input: 'tfjs/tfjs.esm.d.ts' });
|
||||
copy('tfjs/tfjs.esm.d.ts', 'types/lib/dist/tfjs.esm.d.ts');
|
||||
log.state('Copy:', { input: 'src/tfjs/tfjs.esm.d.ts', output: 'dist/tfjs.esm.d.ts' });
|
||||
copy('src/tfjs/tfjs.esm.d.ts', 'dist/tfjs.esm.d.ts');
|
||||
// run api-extractor to create typedef rollup
|
||||
const extractorConfig = APIExtractor.ExtractorConfig.loadFileAndPrepare('.api-extractor.json');
|
||||
const extractorResult = APIExtractor.Extractor.invoke(extractorConfig, {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -234,7 +234,7 @@ async function detectFace() {
|
|||
if (!current?.face?.tensor || !current?.face?.embedding) return false;
|
||||
console.log('face record:', current.face); // eslint-disable-line no-console
|
||||
log(`detected face: ${current.face.gender} ${current.face.age || 0}y distance ${current.face.iris || 0}cm/${Math.round(100 * (current.face.iris || 0) / 2.54) / 100}in`);
|
||||
human.tf.browser.toPixels(current.face.tensor as unknown as H.TensorLike, dom.canvas);
|
||||
await human.tf.browser.toPixels(current.face.tensor, dom.canvas);
|
||||
if (await indexDb.count() === 0) {
|
||||
log('face database is empty: nothing to compare face with');
|
||||
document.body.style.background = 'black';
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -42,7 +42,7 @@ const log = (...msg) => { // helper method to output messages
|
|||
console.log(...msg); // eslint-disable-line no-console
|
||||
};
|
||||
const status = (msg) => dom.fps.innerText = msg; // print status element
|
||||
const perf = (msg) => dom.perf.innerText = 'tensors:' + (human.tf.memory().numTensors as number).toString() + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
|
||||
const perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors.toString() + ' | performance: ' + JSON.stringify(msg).replace(/"|{|}/g, '').replace(/,/g, ' | '); // print performance element
|
||||
|
||||
async function detectionLoop() { // main detection loop
|
||||
if (!dom.video.paused) {
|
||||
|
|
|
|||
28
package.json
28
package.json
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@vladmandic/human",
|
||||
"version": "2.11.1",
|
||||
"version": "3.0.0",
|
||||
"description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition",
|
||||
"sideEffects": false,
|
||||
"main": "dist/human.node.js",
|
||||
|
|
@ -68,21 +68,21 @@
|
|||
"devDependencies": {
|
||||
"@html-eslint/eslint-plugin": "^0.15.0",
|
||||
"@html-eslint/parser": "^0.15.0",
|
||||
"@microsoft/api-extractor": "^7.33.1",
|
||||
"@tensorflow/tfjs": "^3.21.0",
|
||||
"@tensorflow/tfjs-backend-cpu": "^3.21.0",
|
||||
"@tensorflow/tfjs-backend-wasm": "^3.21.0",
|
||||
"@tensorflow/tfjs-backend-webgl": "^3.21.0",
|
||||
"@microsoft/api-extractor": "^7.33.2",
|
||||
"@tensorflow/tfjs": "^4.0.0",
|
||||
"@tensorflow/tfjs-backend-cpu": "^4.0.0",
|
||||
"@tensorflow/tfjs-backend-wasm": "^4.0.0",
|
||||
"@tensorflow/tfjs-backend-webgl": "^4.0.0",
|
||||
"@tensorflow/tfjs-backend-webgpu": "0.0.1-alpha.14",
|
||||
"@tensorflow/tfjs-converter": "^3.21.0",
|
||||
"@tensorflow/tfjs-core": "^3.21.0",
|
||||
"@tensorflow/tfjs-data": "^3.21.0",
|
||||
"@tensorflow/tfjs-layers": "^3.21.0",
|
||||
"@tensorflow/tfjs-node": "^3.21.1",
|
||||
"@tensorflow/tfjs-node-gpu": "^3.21.0",
|
||||
"@tensorflow/tfjs-converter": "^4.0.0",
|
||||
"@tensorflow/tfjs-core": "^4.0.0",
|
||||
"@tensorflow/tfjs-data": "^4.0.0",
|
||||
"@tensorflow/tfjs-layers": "^4.0.0",
|
||||
"@tensorflow/tfjs-node": "^4.0.0",
|
||||
"@tensorflow/tfjs-node-gpu": "^4.0.0",
|
||||
"@tensorflow/tfjs-tflite": "0.0.1-alpha.9",
|
||||
"@types/emscripten": "^1.39.6",
|
||||
"@types/node": "^18.8.5",
|
||||
"@types/node": "^18.11.0",
|
||||
"@types/offscreencanvas": "^2019.7.0",
|
||||
"@typescript-eslint/eslint-plugin": "^5.40.0",
|
||||
"@typescript-eslint/parser": "^5.40.0",
|
||||
|
|
@ -91,7 +91,7 @@
|
|||
"@vladmandic/tfjs": "github:vladmandic/tfjs",
|
||||
"@webgpu/types": "^0.1.22",
|
||||
"canvas": "^2.10.1",
|
||||
"esbuild": "^0.15.10",
|
||||
"esbuild": "^0.15.11",
|
||||
"eslint": "8.25.0",
|
||||
"eslint-config-airbnb-base": "^15.0.0",
|
||||
"eslint-plugin-html": "^7.1.0",
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
* BlazePose model implementation
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import { log, now } from '../util/util';
|
||||
import type { BodyKeypoint, BodyResult, BodyLandmark, Box, Point, BodyAnnotation } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import * as coords from './blazeposecoords';
|
||||
import * as detect from './blazeposedetector';
|
||||
|
|
@ -59,7 +59,7 @@ export async function load(config: Config): Promise<[GraphModel | null, GraphMod
|
|||
return [models.detector, models.landmarks];
|
||||
}
|
||||
|
||||
function prepareImage(input: Tensor, size: number): Tensor {
|
||||
function prepareImage(input: Tensor4D, size: number): Tensor {
|
||||
const t: Record<string, Tensor> = {};
|
||||
if (!input?.shape?.[1] || !input?.shape?.[2]) return input;
|
||||
let final: Tensor;
|
||||
|
|
@ -82,10 +82,10 @@ function prepareImage(input: Tensor, size: number): Tensor {
|
|||
[0, 0], // dont touch rbg
|
||||
];
|
||||
t.pad = tf.pad(t.cropped || input, padding); // use cropped box if it exists
|
||||
t.resize = tf.image.resizeBilinear(t.pad, [size, size]);
|
||||
t.resize = tf.image.resizeBilinear(t.pad as Tensor4D, [size, size]);
|
||||
final = tf.div(t.resize, constants.tf255);
|
||||
} else if (input.shape[1] !== size) { // if input needs resizing
|
||||
t.resize = tf.image.resizeBilinear(t.cropped || input, [size, size]);
|
||||
t.resize = tf.image.resizeBilinear(t.cropped as Tensor4D || input, [size, size]);
|
||||
final = tf.div(t.resize, constants.tf255);
|
||||
} else { // if input is already in a correct resolution just normalize it
|
||||
final = tf.div(t.cropped || input, constants.tf255);
|
||||
|
|
@ -142,7 +142,7 @@ async function detectLandmarks(input: Tensor, config: Config, outputSize: [numbe
|
|||
*/
|
||||
if (!models.landmarks?.['executor']) return null;
|
||||
const t: Record<string, Tensor> = {};
|
||||
[t.ld/* 1,195(39*5) */, t.segmentation/* 1,256,256,1 */, t.heatmap/* 1,64,64,39 */, t.world/* 1,117(39*3) */, t.poseflag/* 1,1 */] = models.landmarks?.execute(input, outputNodes.landmarks) as Tensor[]; // run model
|
||||
[t.ld/* 1,195(39*5) */, t.segmentation/* 1,256,256,1 */, t.heatmap/* 1,64,64,39 */, t.world/* 1,117(39*3) */, t.poseflag/* 1,1 */] = models.landmarks?.execute(input, outputNodes.landmarks) as unknown as Tensor[]; // run model
|
||||
const poseScore = (await t.poseflag.data())[0];
|
||||
const points = await t.ld.data();
|
||||
const distances = await t.world.data();
|
||||
|
|
@ -207,7 +207,7 @@ async function detectBoxes(input: Tensor, config: Config, outputSize: [number, n
|
|||
}
|
||||
*/
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<BodyResult[]> {
|
||||
const outputSize: [number, number] = [input.shape[2] || 0, input.shape[1] || 0];
|
||||
const skipTime = (config.body.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.body.skipFrames || 0);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Tensor } from '../tfjs/types';
|
||||
import type { Box } from '../result';
|
||||
import type { Config } from '../config';
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
* Based on: [**EfficientPose**](https://github.com/daniegr/EfficientPose)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import * as coords from './efficientposecoords';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { BodyResult, Point, BodyLandmark, BodyAnnotation } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -50,8 +50,8 @@ async function max2d(inputs, minScore): Promise<[number, number, number]> {
|
|||
return [0, 0, newScore];
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config): Promise<BodyResult[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
export async function predict(image: Tensor4D, config: Config): Promise<BodyResult[]> {
|
||||
if (!model?.['executor'] || !model?.inputs[0].shape) return [];
|
||||
const skipTime = (config.body.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.body.skipFrames || 0);
|
||||
if (config.skipAllowed && skipTime && skipFrame && Object.keys(cache.keypoints).length > 0) {
|
||||
|
|
@ -61,8 +61,7 @@ export async function predict(image: Tensor, config: Config): Promise<BodyResult
|
|||
skipped = 0;
|
||||
return new Promise(async (resolve) => {
|
||||
const tensor = tf.tidy(() => {
|
||||
if (!model?.inputs[0].shape) return null;
|
||||
const resize = tf.image.resizeBilinear(image, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const resize = tf.image.resizeBilinear(image, [model?.inputs[0].shape?.[2] || 0, model?.inputs[0].shape?.[1] || 0], false);
|
||||
const enhance = tf.mul(resize, constants.tf2);
|
||||
const norm = tf.sub(enhance, constants.tf1);
|
||||
return norm;
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
* Based on: [**MoveNet**](https://blog.tensorflow.org/2021/05/next-generation-pose-detection-with-movenet-and-tensorflowjs.html)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as box from '../util/box';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as coords from './movenetcoords';
|
||||
import * as fix from './movenetfix';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { BodyKeypoint, BodyResult } from '../result';
|
||||
import * as box from '../util/box';
|
||||
import * as coords from './movenetcoords';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { Tensor } from '../tfjs/types';
|
||||
import type { Tensor, Tensor3D } from '../tfjs/types';
|
||||
|
||||
const maxJitter = 0.005; // default allowed jitter is within 0.5%
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ export function padInput(input: Tensor, inputSize: number): Tensor {
|
|||
[0, 0], // dont touch rbg
|
||||
];
|
||||
t.pad = tf.pad(input, cache.padding);
|
||||
t.resize = tf.image.resizeBilinear(t.pad, [inputSize, inputSize]);
|
||||
t.resize = tf.image.resizeBilinear(t.pad as Tensor3D, [inputSize, inputSize]);
|
||||
const final = tf.cast(t.resize, 'int32');
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return final;
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@
|
|||
* Based on: [**PoseNet**](https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { BodyResult, BodyLandmark, Box } from '../result';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, GraphModel, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
import * as utils from './posenetutils';
|
||||
|
|
@ -155,7 +155,7 @@ export function decode(offsets, scores, displacementsFwd, displacementsBwd, maxD
|
|||
return poses;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<BodyResult[]> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<BodyResult[]> {
|
||||
/** posenet is mostly obsolete
|
||||
* caching is not implemented
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ export * from './config';
|
|||
/* Export results details */
|
||||
export * from './result';
|
||||
|
||||
/* Explict reexport of main @tensorflow/tfjs types */
|
||||
export type { Tensor, TensorLike, GraphModel, Rank } from './tfjs/types';
|
||||
/**
|
||||
* Explict reexport of main @tensorflow/tfjs types
|
||||
*/
|
||||
export type { Tensor, Tensor4D, GraphModel, Rank } from './tfjs/types';
|
||||
|
||||
// re-export types
|
||||
export type { DrawOptions } from './draw/options';
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
* Anti-spoofing model implementation
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<number> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<number> {
|
||||
if (!model || !model?.['executor']) return 0;
|
||||
const skipTime = (config.face.antispoof?.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.face.antispoof?.skipFrames || 0);
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@
|
|||
* See `facemesh.ts` for entry point
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as util from './facemeshutil';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Config } from '../config';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, GraphModel, Tensor1D, Tensor2D, Tensor4D } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
import type { Point } from '../result';
|
||||
|
||||
|
|
@ -35,6 +35,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
}
|
||||
|
||||
function decodeBoxes(boxOutputs: Tensor) {
|
||||
if (!anchors || !inputSizeT) return tf.zeros([0, 0]);
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.boxStarts = tf.slice(boxOutputs, [0, 1], [-1, 2]);
|
||||
t.centers = tf.add(t.boxStarts, anchors);
|
||||
|
|
@ -46,12 +47,12 @@ function decodeBoxes(boxOutputs: Tensor) {
|
|||
t.ends = tf.add(t.centersNormalized, t.halfBoxSize);
|
||||
t.startNormalized = tf.mul(t.starts, inputSizeT);
|
||||
t.endNormalized = tf.mul(t.ends, inputSizeT);
|
||||
const boxes = tf.concat2d([t.startNormalized, t.endNormalized], 1);
|
||||
const boxes = tf.concat2d([t.startNormalized as Tensor2D, t.endNormalized as Tensor2D], 1);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return boxes;
|
||||
}
|
||||
|
||||
export async function getBoxes(inputImage: Tensor, config: Config) {
|
||||
export async function getBoxes(inputImage: Tensor4D, config: Config) {
|
||||
// sanity check on input
|
||||
if ((!inputImage) || (inputImage['isDisposedInternal']) || (inputImage.shape.length !== 4) || (inputImage.shape[1] < 1) || (inputImage.shape[2] < 1)) return [];
|
||||
const t: Record<string, Tensor> = {};
|
||||
|
|
@ -64,7 +65,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
|
|||
t.concat384 = tf.concat([sorted[0], sorted[2]], 2); // dim: 384, 1 + 16
|
||||
t.concat512 = tf.concat([sorted[1], sorted[3]], 2); // dim: 512, 1 + 16
|
||||
t.concat = tf.concat([t.concat512, t.concat384], 1);
|
||||
t.batch = tf.squeeze(t.concat, 0);
|
||||
t.batch = tf.squeeze(t.concat, [0]);
|
||||
} else if (Array.isArray(res)) { // new facemesh-detection tfhub model
|
||||
t.batch = tf.squeeze(res[0]);
|
||||
} else { // original blazeface tfhub model
|
||||
|
|
@ -75,7 +76,7 @@ export async function getBoxes(inputImage: Tensor, config: Config) {
|
|||
t.logits = tf.slice(t.batch, [0, 0], [-1, 1]);
|
||||
t.sigmoid = tf.sigmoid(t.logits);
|
||||
t.scores = tf.squeeze(t.sigmoid);
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes as Tensor2D, t.scores as Tensor1D, (config.face.detector?.maxDetected || 0), (config.face.detector?.iouThreshold || 0), (config.face.detector?.minConfidence || 0));
|
||||
const nms = await t.nms.array() as number[];
|
||||
const boxes: DetectBox[] = [];
|
||||
const scores = await t.scores.data();
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
* Uses FaceMesh, Emotion and FaceRes models to create a unified pipeline
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import { env } from '../util/env';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as facemesh from './facemesh';
|
||||
import * as emotion from '../gear/emotion';
|
||||
import * as faceres from './faceres';
|
||||
|
|
@ -18,13 +18,13 @@ import * as ssrnetGender from '../gear/ssrnet-gender';
|
|||
import * as mobilefacenet from './mobilefacenet';
|
||||
import * as insightface from './insightface';
|
||||
import type { FaceResult, Emotion, Gender, Race } from '../result';
|
||||
import type { Tensor } from '../tfjs/types';
|
||||
import type { Tensor4D } from '../tfjs/types';
|
||||
import type { Human } from '../human';
|
||||
import { calculateFaceAngle } from './angles';
|
||||
|
||||
interface DescRes { age: number, gender: Gender, genderScore: number, descriptor: number[], race?: { score: number, race: Race }[] }
|
||||
|
||||
export const detectFace = async (instance: Human /* instance of human */, input: Tensor): Promise<FaceResult[]> => {
|
||||
export const detectFace = async (instance: Human /* instance of human */, input: Tensor4D): Promise<FaceResult[]> => {
|
||||
// run facemesh, includes blazeface and iris
|
||||
let timeStamp: number = now();
|
||||
let ageRes: { age: number } | Promise<{ age: number }> | null;
|
||||
|
|
@ -68,11 +68,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run emotion, inherits face from blazeface
|
||||
instance.analyze('Start Emotion:');
|
||||
if (instance.config.async) {
|
||||
emotionRes = instance.config.face.emotion?.enabled ? emotion.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : [];
|
||||
emotionRes = instance.config.face.emotion?.enabled ? emotion.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : [];
|
||||
} else {
|
||||
instance.state = 'run:emotion';
|
||||
timeStamp = now();
|
||||
emotionRes = instance.config.face.emotion?.enabled ? await emotion.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : [];
|
||||
emotionRes = instance.config.face.emotion?.enabled ? await emotion.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : [];
|
||||
instance.performance.emotion = env.perfadd ? (instance.performance.emotion || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End Emotion:');
|
||||
|
|
@ -80,11 +80,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run antispoof, inherits face from blazeface
|
||||
instance.analyze('Start AntiSpoof:');
|
||||
if (instance.config.async) {
|
||||
antispoofRes = instance.config.face.antispoof?.enabled ? antispoof.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
antispoofRes = instance.config.face.antispoof?.enabled ? antispoof.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
} else {
|
||||
instance.state = 'run:antispoof';
|
||||
timeStamp = now();
|
||||
antispoofRes = instance.config.face.antispoof?.enabled ? await antispoof.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
antispoofRes = instance.config.face.antispoof?.enabled ? await antispoof.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
instance.performance.antispoof = env.perfadd ? (instance.performance.antispoof || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End AntiSpoof:');
|
||||
|
|
@ -92,11 +92,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run liveness, inherits face from blazeface
|
||||
instance.analyze('Start Liveness:');
|
||||
if (instance.config.async) {
|
||||
livenessRes = instance.config.face.liveness?.enabled ? liveness.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
livenessRes = instance.config.face.liveness?.enabled ? liveness.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
} else {
|
||||
instance.state = 'run:liveness';
|
||||
timeStamp = now();
|
||||
livenessRes = instance.config.face.liveness?.enabled ? await liveness.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
livenessRes = instance.config.face.liveness?.enabled ? await liveness.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : 0;
|
||||
instance.performance.liveness = env.perfadd ? (instance.performance.antispoof || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End Liveness:');
|
||||
|
|
@ -104,11 +104,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run gear, inherits face from blazeface
|
||||
instance.analyze('Start GEAR:');
|
||||
if (instance.config.async) {
|
||||
gearRes = instance.config.face.gear?.enabled ? gear.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
gearRes = instance.config.face.gear?.enabled ? gear.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
} else {
|
||||
instance.state = 'run:gear';
|
||||
timeStamp = now();
|
||||
gearRes = instance.config.face.gear?.enabled ? await gear.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
gearRes = instance.config.face.gear?.enabled ? await gear.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
instance.performance.gear = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End GEAR:');
|
||||
|
|
@ -116,13 +116,13 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run gear, inherits face from blazeface
|
||||
instance.analyze('Start SSRNet:');
|
||||
if (instance.config.async) {
|
||||
ageRes = instance.config.face['ssrnet']?.enabled ? ssrnetAge.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
genderRes = instance.config.face['ssrnet']?.enabled ? ssrnetGender.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
ageRes = instance.config.face['ssrnet']?.enabled ? ssrnetAge.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
genderRes = instance.config.face['ssrnet']?.enabled ? ssrnetGender.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
} else {
|
||||
instance.state = 'run:ssrnet';
|
||||
timeStamp = now();
|
||||
ageRes = instance.config.face['ssrnet']?.enabled ? await ssrnetAge.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
genderRes = instance.config.face['ssrnet']?.enabled ? await ssrnetGender.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
ageRes = instance.config.face['ssrnet']?.enabled ? await ssrnetAge.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
genderRes = instance.config.face['ssrnet']?.enabled ? await ssrnetGender.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
instance.performance.ssrnet = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End SSRNet:');
|
||||
|
|
@ -130,11 +130,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run mobilefacenet alternative, inherits face from blazeface
|
||||
instance.analyze('Start MobileFaceNet:');
|
||||
if (instance.config.async) {
|
||||
mobilefacenetRes = instance.config.face['mobilefacenet']?.enabled ? mobilefacenet.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
mobilefacenetRes = instance.config.face['mobilefacenet']?.enabled ? mobilefacenet.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
} else {
|
||||
instance.state = 'run:mobilefacenet';
|
||||
timeStamp = now();
|
||||
mobilefacenetRes = instance.config.face['mobilefacenet']?.enabled ? await mobilefacenet.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
mobilefacenetRes = instance.config.face['mobilefacenet']?.enabled ? await mobilefacenet.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
instance.performance.mobilefacenet = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End MobileFaceNet:');
|
||||
|
|
@ -142,11 +142,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run insightface alternative, inherits face from blazeface
|
||||
instance.analyze('Start InsightFace:');
|
||||
if (instance.config.async) {
|
||||
insightfaceRes = instance.config.face['insightface']?.enabled ? insightface.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
insightfaceRes = instance.config.face['insightface']?.enabled ? insightface.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
} else {
|
||||
instance.state = 'run:mobilefacenet';
|
||||
timeStamp = now();
|
||||
insightfaceRes = instance.config.face['insightface']?.enabled ? await insightface.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
insightfaceRes = instance.config.face['insightface']?.enabled ? await insightface.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length) : null;
|
||||
instance.performance.mobilefacenet = Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End InsightFace:');
|
||||
|
|
@ -154,11 +154,11 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
// run faceres, inherits face from blazeface
|
||||
instance.analyze('Start Description:');
|
||||
if (instance.config.async) {
|
||||
descRes = faceres.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length);
|
||||
descRes = faceres.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length);
|
||||
} else {
|
||||
instance.state = 'run:description';
|
||||
timeStamp = now();
|
||||
descRes = await faceres.predict(faces[i].tensor || tf.tensor([]), instance.config, i, faces.length);
|
||||
descRes = await faceres.predict(faces[i].tensor as Tensor4D || tf.tensor([]), instance.config, i, faces.length);
|
||||
instance.performance.description = env.perfadd ? (instance.performance.description || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
|
||||
}
|
||||
instance.analyze('End Description:');
|
||||
|
|
@ -207,7 +207,7 @@ export const detectFace = async (instance: Human /* instance of human */, input:
|
|||
: 0; // note: average human iris size is 11.7mm
|
||||
|
||||
// optionally return tensor
|
||||
const tensor = instance.config.face.detector?.return ? tf.squeeze(faces[i].tensor) : null;
|
||||
const tensor = instance.config.face.detector?.return ? tf.squeeze(faces[i].tensor as Tensor4D) : null;
|
||||
// dispose original face tensor
|
||||
tf.dispose(faces[i].tensor);
|
||||
// delete temp face image
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// https://github.com/TropComplique/FaceBoxes-tensorflow
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
|
|
@ -29,14 +29,14 @@ export class FaceBoxes {
|
|||
const [scoresT, boxesT, numT] = await this.model.executeAsync(castT) as Tensor[];
|
||||
const scores = await scoresT.data();
|
||||
const squeezeT = tf.squeeze(boxesT);
|
||||
const boxes = squeezeT.arraySync();
|
||||
const boxes = squeezeT.arraySync() as number[][];
|
||||
scoresT.dispose();
|
||||
boxesT.dispose();
|
||||
squeezeT.dispose();
|
||||
numT.dispose();
|
||||
castT.dispose();
|
||||
resizeT.dispose();
|
||||
for (const i in boxes) {
|
||||
for (let i = 0; i < boxes.length; i++) {
|
||||
if (scores[i] && scores[i] > (this.config.face.detector?.minConfidence || 0.1)) {
|
||||
const crop = [boxes[i][0] / this.enlarge, boxes[i][1] / this.enlarge, boxes[i][2] * this.enlarge, boxes[i][3] * this.enlarge];
|
||||
const boxRaw: Box = [crop[1], crop[0], (crop[3]) - (crop[1]), (crop[2]) - (crop[0])];
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@
|
|||
* - Eye Iris Details: [**MediaPipe Iris**](https://drive.google.com/file/d/1bsWbokp9AklH2ANjCfmjqEzzxO1CNbMu/view)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as blazeface from './blazeface';
|
||||
import * as util from './facemeshutil';
|
||||
import * as coords from './facemeshcoords';
|
||||
|
|
@ -17,7 +17,7 @@ import * as iris from './iris';
|
|||
import * as attention from './attention';
|
||||
import { histogramEqualization } from '../image/enhance';
|
||||
import { env } from '../util/env';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import type { FaceResult, FaceLandmark, Point } from '../result';
|
||||
import type { Config } from '../config';
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ const cache = {
|
|||
let model: GraphModel | null = null;
|
||||
let inputSize = 0;
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<FaceResult[]> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<FaceResult[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
// reset cached boxes
|
||||
const skipTime = (config.face.detector?.skipTime || 0) > (now() - cache.timestamp);
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* See `facemesh.ts` for entry point
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as coords from './facemeshcoords';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Box, Point } from '../result';
|
||||
|
|
@ -175,7 +175,7 @@ export function correctFaceRotation(rotate, box, input, inputSize) {
|
|||
if (largeAngle) { // perform rotation only if angle is sufficiently high
|
||||
const center: Point = getBoxCenter(box);
|
||||
const centerRaw: Point = [center[0] / input.shape[2], center[1] / input.shape[1]];
|
||||
const rotated = tf.image.rotateWithOffset(input, angle, 0, centerRaw);
|
||||
const rotated = tf.image.rotateWithOffset(input, angle, 0, [centerRaw[0], centerRaw[1]]);
|
||||
rotationMatrix = buildRotationMatrix(-angle, center);
|
||||
face = cutAndResize(box, rotated, [inputSize, inputSize]);
|
||||
tf.dispose(rotated);
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@
|
|||
* Based on: [**HSE-FaceRes**](https://github.com/HSE-asavchenko/HSE_FaceRec_tf)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import { env } from '../util/env';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, GraphModel, Tensor4D, Tensor1D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import type { Gender, Race } from '../result';
|
||||
|
||||
|
|
@ -33,7 +33,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
}
|
||||
|
||||
export function enhance(input): Tensor {
|
||||
const tensor = (input.image || input.tensor || input) as Tensor; // input received from detector is already normalized to 0..1, input is also assumed to be straightened
|
||||
const tensor = (input.image || input.tensor || input) as Tensor4D; // input received from detector is already normalized to 0..1, input is also assumed to be straightened
|
||||
if (!model?.inputs[0].shape) return tensor; // model has no shape so no point continuing
|
||||
const crop: Tensor = tf.image.resizeBilinear(tensor, [model.inputs[0].shape[2], model.inputs[0].shape[1]], false);
|
||||
const norm: Tensor = tf.mul(crop, constants.tf255);
|
||||
|
|
@ -58,7 +58,7 @@ export function enhance(input): Tensor {
|
|||
*/
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<FaceRes> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<FaceRes> {
|
||||
const obj: FaceRes = {
|
||||
age: 0 as number,
|
||||
gender: 'unknown' as Gender,
|
||||
|
|
@ -86,7 +86,7 @@ export async function predict(image: Tensor, config: Config, idx: number, count:
|
|||
obj.gender = gender[0] <= 0.5 ? 'female' : 'male';
|
||||
obj.genderScore = Math.min(0.99, confidence);
|
||||
}
|
||||
const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100), 1);
|
||||
const argmax = tf.argMax(resT.find((t) => t.shape[1] === 100) as Tensor1D, 1);
|
||||
const ageIdx: number = (await argmax.data())[0];
|
||||
tf.dispose(argmax);
|
||||
const ageT = resT.find((t) => t.shape[1] === 100) as Tensor;
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@
|
|||
* Alternative face embedding detection
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, Tensor4D, GraphModel } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config, idx, count): Promise<number[]> {
|
||||
export async function predict(input: Tensor4D, config: Config, idx, count): Promise<number[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
const skipFrame = skipped < (config.face['insightface']?.skipFrames || 0);
|
||||
const skipTime = (config.face['insightface']?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as coords from './facemeshcoords';
|
||||
import * as util from './facemeshutil';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
import { log } from '../util/util';
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
* Anti-spoofing model implementation
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
|
||||
let model: GraphModel | null;
|
||||
|
|
@ -22,7 +22,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<number> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<number> {
|
||||
if (!model?.['executor']) return 0;
|
||||
const skipTime = (config.face.liveness?.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.face.liveness?.skipFrames || 0);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import type { Tensor } from '../tfjs/types';
|
||||
import type { FaceResult } from '../result';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
// import * as tf from 'dist/tfjs.esm.js';
|
||||
import { meshAnnotations } from './facemeshcoords';
|
||||
|
||||
const expandFact = 0.1;
|
||||
|
|
@ -36,6 +36,6 @@ export async function mask(face: FaceResult): Promise<Tensor | undefined> {
|
|||
}
|
||||
}
|
||||
const output = buffer.toTensor();
|
||||
tf.dispose(buffer);
|
||||
// tf.dispose(buffer);
|
||||
return output;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@
|
|||
* Obsolete and replaced by `faceres` that performs age/gender/descriptor analysis
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, Tensor4D, GraphModel } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ const factor = 5;
|
|||
const contrast = merge.sub(mean).mul(factor).add(mean);
|
||||
*/
|
||||
|
||||
export async function predict(input: Tensor, config: Config, idx, count): Promise<number[]> {
|
||||
export async function predict(input: Tensor4D, config: Config, idx, count): Promise<number[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
const skipFrame = skipped < (config.face['mobilefacenet']?.skipFrames || 0);
|
||||
const skipTime = (config.face['mobilefacenet']?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@
|
|||
* [**Oarriaga**](https://github.com/oarriaga/face_classification)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Emotion } from '../result';
|
||||
import { log, now } from '../util/util';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { env } from '../util/env';
|
||||
import { constants } from '../tfjs/constants';
|
||||
|
|
@ -27,7 +27,7 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<{ score: number, emotion: Emotion }[]> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<{ score: number, emotion: Emotion }[]> {
|
||||
if (!model) return [];
|
||||
const skipFrame = skipped < (config.face.emotion?.skipFrames || 0);
|
||||
const skipTime = (config.face.emotion?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
* Based on: [**GEAR Predictor**](https://github.com/Udolf15/GEAR-Predictor)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { Gender, Race } from '../result';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
|
||||
export interface GearType { age: number, gender: Gender, genderScore: number, race: { score: number, race: Race }[] }
|
||||
|
|
@ -28,7 +28,7 @@ export async function load(config: Config) {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<GearType> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<GearType> {
|
||||
if (!model) return { age: 0, gender: 'unknown', genderScore: 0, race: [] };
|
||||
const skipFrame = skipped < (config.face.gear?.skipFrames || 0);
|
||||
const skipTime = (config.face.gear?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
* Based on: [**SSR-Net**](https://github.com/shamangary/SSR-Net)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { env } from '../util/env';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
|
||||
let model: GraphModel | null;
|
||||
const last: { age: number }[] = [];
|
||||
|
|
@ -25,7 +25,7 @@ export async function load(config: Config) {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx: number, count: number): Promise<{ age: number }> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx: number, count: number): Promise<{ age: number }> {
|
||||
if (!model) return { age: 0 };
|
||||
const skipFrame = skipped < (config.face['ssrnet']?.skipFrames || 0);
|
||||
const skipTime = (config.face['ssrnet']?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
* Based on: [**SSR-Net**](https://github.com/shamangary/SSR-Net)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Gender } from '../result';
|
||||
import type { Config } from '../config';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
|
||||
let model: GraphModel | null;
|
||||
|
|
@ -29,7 +29,7 @@ export async function load(config: Config) {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config, idx, count): Promise<{ gender: Gender, genderScore: number }> {
|
||||
export async function predict(image: Tensor4D, config: Config, idx, count): Promise<{ gender: Gender, genderScore: number }> {
|
||||
if (!model) return { gender: 'unknown', genderScore: 0 };
|
||||
const skipFrame = skipped < (config.face['ssrnet']?.skipFrames || 0);
|
||||
const skipTime = (config.face['ssrnet']?.skipTime || 0) > (now() - lastTime);
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
* See `handpose.ts` for entry point
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as util from './handposeutil';
|
||||
import * as anchors from './handposeanchors';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { Tensor, GraphModel } from '../tfjs/types';
|
||||
import type { Tensor, Tensor1D, Tensor2D, Tensor4D, GraphModel } from '../tfjs/types';
|
||||
import type { Point } from '../result';
|
||||
import type { Config } from '../config';
|
||||
|
||||
|
|
@ -39,22 +39,22 @@ export class HandDetector {
|
|||
t.startPoints = tf.mul(t.sub, this.inputSizeTensor);
|
||||
t.add = tf.add(t.boxCenterPoints, t.halfBoxSizes);
|
||||
t.endPoints = tf.mul(t.add, this.inputSizeTensor);
|
||||
const res = tf.concat2d([t.startPoints, t.endPoints], 1);
|
||||
const res = tf.concat2d([t.startPoints as Tensor2D, t.endPoints as Tensor2D], 1);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return res as Tensor;
|
||||
}
|
||||
|
||||
normalizeLandmarks(rawPalmLandmarks, index: number) {
|
||||
normalizeLandmarks(rawPalmLandmarks, index: number): Tensor {
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.reshape = tf.reshape(rawPalmLandmarks, [-1, 7, 2]);
|
||||
t.div = tf.div(t.reshape, this.inputSizeTensor);
|
||||
t.landmarks = tf.add(t.div, this.anchors[index] ? this.anchors[index] : 0);
|
||||
const res = tf.mul(t.landmarks, this.inputSizeTensor);
|
||||
Object.keys(t).forEach((tensor) => tf.dispose(t[tensor]));
|
||||
return res as Tensor;
|
||||
return res;
|
||||
}
|
||||
|
||||
async predict(input: Tensor, config: Config): Promise<{ startPoint: Point; endPoint: Point, palmLandmarks: Point[]; confidence: number }[]> {
|
||||
async predict(input: Tensor4D, config: Config): Promise<{ startPoint: Point; endPoint: Point, palmLandmarks: Point[]; confidence: number }[]> {
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.resize = tf.image.resizeBilinear(input, [this.inputSize, this.inputSize]);
|
||||
t.div = tf.div(t.resize, constants.tf127);
|
||||
|
|
@ -68,7 +68,7 @@ export class HandDetector {
|
|||
t.boxes = tf.slice(t.predictions, [0, 1], [-1, 4]);
|
||||
t.norm = this.normalizeBoxes(t.boxes);
|
||||
// box detection is flaky so we look for 3x boxes than we need results
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm, t.scores, 3 * (config.hand?.maxDetected || 1), config.hand.iouThreshold, config.hand.minConfidence);
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.norm as Tensor2D, t.scores as Tensor1D, 3 * (config.hand?.maxDetected || 1), config.hand.iouThreshold, config.hand.minConfidence);
|
||||
const nms = await t.nms.array() as number[];
|
||||
const hands: { startPoint: Point; endPoint: Point; palmLandmarks: Point[]; confidence: number }[] = [];
|
||||
for (const index of nms) {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* See `handpose.ts` for entry point
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as util from './handposeutil';
|
||||
import type * as detector from './handposedetector';
|
||||
import { constants } from '../tfjs/constants';
|
||||
|
|
@ -115,7 +115,7 @@ export class HandPipeline {
|
|||
if (config.hand.landmarks) {
|
||||
const angle = config.hand.rotation ? util.computeRotation(currentBox.palmLandmarks[palmLandmarksPalmBase], currentBox.palmLandmarks[palmLandmarksMiddleFingerBase]) : 0;
|
||||
const palmCenter = util.getBoxCenter(currentBox);
|
||||
const palmCenterNormalized = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
|
||||
const palmCenterNormalized: [number, number] = [palmCenter[0] / image.shape[2], palmCenter[1] / image.shape[1]];
|
||||
const rotatedImage = config.hand.rotation && env.kernels.includes('rotatewithoffset') ? tf.image.rotateWithOffset(image, angle, 0, palmCenterNormalized) : image.clone();
|
||||
const rotationMatrix = util.buildRotationMatrix(-angle, palmCenter);
|
||||
const newBox = useFreshBox ? this.getBoxForPalmLandmarks(currentBox.palmLandmarks, rotationMatrix) : currentBox;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Point } from '../result';
|
||||
|
||||
export function getBoxSize(box) {
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@
|
|||
* - Hand Tracking: [**HandTracking**](https://github.com/victordibia/handtracking)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as box from '../util/box';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import type { HandResult, HandType, Box, Point } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor1D, Tensor2D, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
import * as fingerPose from './fingerpose';
|
||||
|
|
@ -100,7 +100,7 @@ export async function load(config: Config): Promise<[GraphModel | null, GraphMod
|
|||
return models;
|
||||
}
|
||||
|
||||
async function detectHands(input: Tensor, config: Config): Promise<HandDetectResult[]> {
|
||||
async function detectHands(input: Tensor4D, config: Config): Promise<HandDetectResult[]> {
|
||||
const hands: HandDetectResult[] = [];
|
||||
if (!input || !models[0]) return hands;
|
||||
const t: Record<string, Tensor> = {};
|
||||
|
|
@ -121,7 +121,7 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
|
|||
t.max = tf.max(t.filtered, 1); // max overall score
|
||||
t.argmax = tf.argMax(t.filtered, 1); // class index of max overall score
|
||||
let id = 0;
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.max, (config.hand.maxDetected || 0) + 1, config.hand.iouThreshold || 0, config.hand.minConfidence || 1);
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes as Tensor2D, t.max as Tensor1D, (config.hand.maxDetected || 0) + 1, config.hand.iouThreshold || 0, config.hand.minConfidence || 1);
|
||||
const nms = await t.nms.data();
|
||||
const scores = await t.max.data();
|
||||
const classNum = await t.argmax.data();
|
||||
|
|
@ -143,7 +143,7 @@ async function detectHands(input: Tensor, config: Config): Promise<HandDetectRes
|
|||
return hands;
|
||||
}
|
||||
|
||||
async function detectFingers(input: Tensor, h: HandDetectResult, config: Config): Promise<HandResult> {
|
||||
async function detectFingers(input: Tensor4D, h: HandDetectResult, config: Config): Promise<HandResult> {
|
||||
const hand: HandResult = { // initial values inherited from hand detect
|
||||
id: h.id,
|
||||
score: Math.round(100 * h.score) / 100,
|
||||
|
|
@ -181,7 +181,7 @@ async function detectFingers(input: Tensor, h: HandDetectResult, config: Config)
|
|||
return hand;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<HandResult[]> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<HandResult[]> {
|
||||
if (!models[0]?.['executor'] || !models[1]?.['executor'] || !models[0].inputs[0].shape || !models[1].inputs[0].shape) return []; // something is wrong with the model
|
||||
outputSize = [input.shape[2] || 0, input.shape[1] || 0];
|
||||
skipped++; // increment skip frames
|
||||
|
|
|
|||
17
src/human.ts
17
src/human.ts
|
|
@ -8,12 +8,12 @@
|
|||
*/
|
||||
|
||||
// module imports
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now, mergeDeep, validate } from './util/util';
|
||||
import { defaults } from './config';
|
||||
import { env, Env } from './util/env';
|
||||
import { WebCam } from './util/webcam';
|
||||
import { setModelLoadOptions } from './tfjs/load';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as app from '../package.json';
|
||||
import * as backend from './tfjs/backend';
|
||||
import * as draw from './draw/draw';
|
||||
|
|
@ -41,9 +41,11 @@ import * as selfie from './segmentation/selfie';
|
|||
import * as warmups from './warmup';
|
||||
|
||||
// type definitions
|
||||
import type { Input, Tensor, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas } from './exports';
|
||||
import type { Input, DrawOptions, Config, Result, FaceResult, HandResult, BodyResult, ObjectResult, GestureResult, PersonResult, AnyCanvas } from './exports';
|
||||
import type { Tensor, Tensor4D } from './tfjs/types';
|
||||
// type exports
|
||||
export * from './exports';
|
||||
// tfjs re-export
|
||||
|
||||
/** **Human** library main class
|
||||
*
|
||||
|
|
@ -324,6 +326,7 @@ export class Human {
|
|||
await tf.ready();
|
||||
if (this.env.browser) {
|
||||
if (this.config.debug) log('configuration:', this.config);
|
||||
// @ts-ignore private property
|
||||
if (this.config.debug) log('tf flags:', this.tf.ENV.flags);
|
||||
}
|
||||
}
|
||||
|
|
@ -379,13 +382,15 @@ export class Human {
|
|||
* - actual detection object can be accessed via `human.result`
|
||||
*/
|
||||
async profile(input: Input, userConfig?: Partial<Config>): Promise<{ kernel: string, time: number, perc: number }[]> {
|
||||
// @ts-ignore profile wraps method return values
|
||||
const profile = await this.tf.profile(() => this.detect(input, userConfig));
|
||||
const kernels: Record<string, number> = {};
|
||||
let total = 0;
|
||||
for (const kernel of profile.kernels) { // sum kernel time values per kernel
|
||||
if (kernels[kernel.name]) kernels[kernel.name] += kernel.kernelTimeMs;
|
||||
else kernels[kernel.name] = kernel.kernelTimeMs;
|
||||
total += kernel.kernelTimeMs;
|
||||
const ms = Number(kernel.kernelTimeMs) || 0;
|
||||
if (kernels[kernel.name]) kernels[kernel.name] += ms;
|
||||
else kernels[kernel.name] = ms;
|
||||
total += ms;
|
||||
}
|
||||
const kernelArr: { kernel: string, time: number, perc: number }[] = [];
|
||||
Object.entries(kernels).forEach((key) => kernelArr.push({ kernel: key[0], time: key[1] as unknown as number, perc: 0 })); // convert to array
|
||||
|
|
@ -434,7 +439,7 @@ export class Human {
|
|||
|
||||
timeStamp = now();
|
||||
this.state = 'image';
|
||||
const img = await image.process(input, this.config) as { canvas: AnyCanvas, tensor: Tensor };
|
||||
const img = await image.process(input, this.config) as { canvas: AnyCanvas, tensor: Tensor4D };
|
||||
this.process = img;
|
||||
this.performance.inputProcess = this.env.perfadd ? (this.performance.inputProcess || 0) + Math.trunc(now() - timeStamp) : Math.trunc(now() - timeStamp);
|
||||
this.analyze('Get Image:');
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Image enhancements
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Tensor } from '../exports';
|
||||
|
||||
export async function histogramEqualization(inputImage: Tensor): Promise<Tensor> {
|
||||
|
|
@ -18,7 +18,7 @@ export async function histogramEqualization(inputImage: Tensor): Promise<Tensor>
|
|||
const fact = [tf.div(maxValue, range[0]), tf.div(maxValue, range[1]), tf.div(maxValue, range[2])];
|
||||
const enh = [tf.mul(sub[0], fact[0]), tf.mul(sub[1], fact[1]), tf.mul(sub[2], fact[2])];
|
||||
const rgb = tf.stack([enh[0], enh[1], enh[2]], 2);
|
||||
const reshape = tf.reshape(rgb, [1, squeeze.shape[0], squeeze.shape[1], 3]);
|
||||
const reshape = tf.reshape(rgb, [1, squeeze.shape[0] || 0, squeeze.shape[1] || 0, 3]);
|
||||
tf.dispose([...channels, ...min, ...max, ...sub, ...range, ...fact, ...enh, rgb, squeeze]);
|
||||
return reshape as Tensor; // output shape is [1, height, width, 3]
|
||||
return reshape; // output shape is [1, height, width, 3]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
* Image Processing algorithm implementation
|
||||
*/
|
||||
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as fxImage from './imagefx';
|
||||
import type { Input, AnyCanvas, Tensor, Config } from '../exports';
|
||||
import type { Input, AnyCanvas, Config } from '../exports';
|
||||
import type { Tensor, Tensor3D, Tensor4D } from '../tfjs/types';
|
||||
import { env } from '../util/env';
|
||||
import { log } from '../util/util';
|
||||
import * as enhance from './enhance';
|
||||
|
|
@ -64,7 +65,7 @@ export function copy(input: AnyCanvas, output?: AnyCanvas) {
|
|||
// process input image and return tensor
|
||||
// input can be tensor, imagedata, htmlimageelement, htmlvideoelement
|
||||
// input is resized and run through imagefx filter
|
||||
export async function process(input: Input, config: Config, getTensor: boolean = true): Promise<{ tensor: Tensor | null, canvas: AnyCanvas | null }> {
|
||||
export async function process(input: Input, config: Config, getTensor: boolean = true): Promise<{ tensor: Tensor4D | null, canvas: AnyCanvas | null }> {
|
||||
if (!input) {
|
||||
// throw new Error('input is missing');
|
||||
if (config.debug) log('input error: input is missing');
|
||||
|
|
@ -88,13 +89,13 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
}
|
||||
if (input instanceof tf.Tensor) { // if input is tensor use as-is without filters but correct shape as needed
|
||||
let tensor: Tensor | null = null;
|
||||
if ((input as Tensor)['isDisposedInternal']) throw new Error('input error: attempted to use tensor but it is disposed');
|
||||
if (input['isDisposedInternal']) throw new Error('input error: attempted to use tensor but it is disposed');
|
||||
if (!(input as Tensor).shape) throw new Error('input error: attempted to use tensor without a shape');
|
||||
if ((input as Tensor).shape.length === 3) { // [height, width, 3 || 4]
|
||||
if ((input as Tensor).shape[2] === 3) { // [height, width, 3] so add batch
|
||||
tensor = tf.expandDims(input, 0);
|
||||
} else if ((input as Tensor).shape[2] === 4) { // [height, width, 4] so strip alpha and add batch
|
||||
const rgb = tf.slice3d(input, [0, 0, 0], [-1, -1, 3]);
|
||||
const rgb = tf.slice3d(input as Tensor3D, [0, 0, 0], [-1, -1, 3]);
|
||||
tensor = tf.expandDims(rgb, 0);
|
||||
tf.dispose(rgb);
|
||||
}
|
||||
|
|
@ -102,7 +103,7 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
if ((input as Tensor).shape[3] === 3) { // [1, width, height, 3] just clone
|
||||
tensor = tf.clone(input);
|
||||
} else if ((input as Tensor).shape[3] === 4) { // [1, width, height, 4] so strip alpha
|
||||
tensor = tf.slice4d(input, [0, 0, 0, 0], [-1, -1, -1, 3]);
|
||||
tensor = tf.slice4d(input as Tensor4D, [0, 0, 0, 0], [-1, -1, -1, 3]);
|
||||
}
|
||||
}
|
||||
// at the end shape must be [1, height, width, 3]
|
||||
|
|
@ -112,7 +113,7 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
tf.dispose(tensor);
|
||||
tensor = cast;
|
||||
}
|
||||
return { tensor, canvas: (config.filter.return ? outCanvas : null) };
|
||||
return { tensor: tensor as Tensor4D, canvas: (config.filter.return ? outCanvas : null) };
|
||||
}
|
||||
// check if resizing will be needed
|
||||
if (typeof input['readyState'] !== 'undefined' && (input as HTMLMediaElement).readyState <= 2) {
|
||||
|
|
@ -204,7 +205,7 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
let depth = 3;
|
||||
if ((typeof ImageData !== 'undefined' && input instanceof ImageData) || ((input as ImageData).data && (input as ImageData).width && (input as ImageData).height)) { // if input is imagedata, just use it
|
||||
if (env.browser && tf.browser) {
|
||||
pixels = tf.browser ? tf.browser.fromPixels(input) : null;
|
||||
pixels = tf.browser ? tf.browser.fromPixels(input as ImageData) : null;
|
||||
} else {
|
||||
depth = (input as ImageData).data.length / (input as ImageData).height / (input as ImageData).width;
|
||||
// const arr = Uint8Array.from(input['data']);
|
||||
|
|
@ -215,10 +216,10 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
if (!tmpCanvas || (outCanvas.width !== tmpCanvas.width) || (outCanvas.height !== tmpCanvas.height)) tmpCanvas = canvas(outCanvas.width, outCanvas.height); // init output canvas
|
||||
if (tf.browser && env.browser) {
|
||||
if (config.backend === 'webgl' || config.backend === 'humangl' || config.backend === 'webgpu') {
|
||||
pixels = tf.browser.fromPixels(outCanvas); // safe to reuse since both backend and context are gl based
|
||||
pixels = tf.browser.fromPixels(outCanvas as HTMLCanvasElement); // safe to reuse since both backend and context are gl based
|
||||
} else {
|
||||
tmpCanvas = copy(outCanvas); // cannot use output canvas as it already has gl context so we do a silly one more canvas
|
||||
pixels = tf.browser.fromPixels(tmpCanvas);
|
||||
pixels = tf.browser.fromPixels(tmpCanvas as HTMLCanvasElement);
|
||||
}
|
||||
} else {
|
||||
const tempCanvas = copy(outCanvas); // cannot use output canvas as it already has gl context so we do a silly one more canvas
|
||||
|
|
@ -238,7 +239,7 @@ export async function process(input: Input, config: Config, getTensor: boolean =
|
|||
const casted: Tensor = tf.cast(pixels, 'float32');
|
||||
const tensor: Tensor = config.filter.equalization ? await enhance.histogramEqualization(casted) : tf.expandDims(casted, 0);
|
||||
tf.dispose([pixels, casted]);
|
||||
return { tensor, canvas: (config.filter.return ? outCanvas : null) };
|
||||
return { tensor: tensor as Tensor4D, canvas: (config.filter.return ? outCanvas : null) };
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -317,7 +318,7 @@ export async function compare(config: Partial<Config>, input1: Tensor, input2: T
|
|||
return 0;
|
||||
}
|
||||
t.input1 = tf.clone(input1);
|
||||
t.input2 = (input1.shape[1] !== input2.shape[1] || input1.shape[2] !== input2.shape[2]) ? tf.image.resizeBilinear(input2, [input1.shape[1], input1.shape[2]]) : tf.clone(input2);
|
||||
t.input2 = (input1.shape[1] !== input2.shape[1] || input1.shape[2] !== input2.shape[2]) ? tf.image.resizeBilinear(input2 as Tensor3D, [input1.shape[1], input1.shape[2]]) : tf.clone(input2);
|
||||
t.diff = tf.sub(t.input1, t.input2);
|
||||
t.squared = tf.mul(t.diff, t.diff);
|
||||
t.sum = tf.sum(t.squared);
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
* Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { labels } from './labels';
|
||||
import type { ObjectResult, ObjectType, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor1D, Tensor2D, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -36,13 +36,13 @@ async function process(res: Tensor | null, outputShape: [number, number], config
|
|||
const results: ObjectResult[] = [];
|
||||
const detections = await res.array() as number[][][];
|
||||
t.squeeze = tf.squeeze(res);
|
||||
const arr = tf.split(t.squeeze, 6, 1) as Tensor[]; // x1, y1, x2, y2, score, class
|
||||
const arr = tf.split(t.squeeze, 6, 1); // x1, y1, x2, y2, score, class
|
||||
t.stack = tf.stack([arr[1], arr[0], arr[3], arr[2]], 1); // reorder dims as tf.nms expects y, x
|
||||
t.boxes = tf.squeeze(t.stack);
|
||||
t.scores = tf.squeeze(arr[4]);
|
||||
t.classes = tf.squeeze(arr[5]);
|
||||
tf.dispose([res, ...arr]);
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes, t.scores, config.object.maxDetected, config.object.iouThreshold, (config.object.minConfidence || 0));
|
||||
t.nms = await tf.image.nonMaxSuppressionAsync(t.boxes as Tensor2D, t.scores as Tensor1D, config.object.maxDetected || 0, config.object.iouThreshold, (config.object.minConfidence || 0));
|
||||
const nms = await t.nms.data();
|
||||
let i = 0;
|
||||
for (const id of Array.from(nms)) {
|
||||
|
|
@ -72,7 +72,7 @@ async function process(res: Tensor | null, outputShape: [number, number], config
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<ObjectResult[]> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<ObjectResult[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
const skipTime = (config.object.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.object.skipFrames || 0);
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
* Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import { labels } from './labels';
|
||||
import type { ObjectResult, ObjectType, Box } from '../result';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor2D, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -39,14 +39,14 @@ async function process(res: Tensor[], outputShape: [number, number], config: Con
|
|||
// find scores, boxes, classes
|
||||
const baseSize = strideSize * 13; // 13x13=169, 26x26=676, 52x52=2704
|
||||
// find boxes and scores output depending on stride
|
||||
const scoresT = tf.squeeze(res.find((a: Tensor) => (a.shape[1] === (baseSize ** 2) && (a.shape[2] || 0) === labels.length)));
|
||||
const scoresT = tf.squeeze(res.find((a) => (a.shape[1] === (baseSize ** 2) && (a.shape[2] || 0) === labels.length)) as Tensor2D);
|
||||
const scores = await scoresT.array(); // optionally use exponential scores or just as-is
|
||||
const featuresT = tf.squeeze(res.find((a: Tensor) => (a.shape[1] === (baseSize ** 2) && (a.shape[2] || 0) < labels.length)));
|
||||
const boxesMaxT = featuresT.reshape([-1, 4, featuresT.shape[1] / 4]); // reshape [output] to [4, output / 4] where number is number of different features inside each stride
|
||||
const boxIdxT = boxesMaxT.argMax(2); // what we need is indexes of features with highest scores, not values itself
|
||||
const featuresT = tf.squeeze(res.find((a) => (a.shape[1] === (baseSize ** 2) && (a.shape[2] || 0) < labels.length)) as Tensor2D);
|
||||
const boxesMaxT = tf.reshape(featuresT, [-1, 4, (featuresT.shape?.[1] || 0) / 4]); // reshape [output] to [4, output / 4] where number is number of different features inside each stride
|
||||
const boxIdxT = tf.argMax(boxesMaxT, 2); // what we need is indexes of features with highest scores, not values itself
|
||||
const boxIdx = await boxIdxT.array(); // what we need is indexes of features with highest scores, not values itself
|
||||
for (let i = 0; i < scoresT.shape[0]; i++) { // total strides (x * y matrix)
|
||||
for (let j = 0; j < scoresT.shape[1]; j++) { // one score for each class
|
||||
for (let j = 0; j < (scoresT.shape?.[1] || 0); j++) { // one score for each class
|
||||
const score = scores[i][j]; // get score for current position
|
||||
if (score > (config.object.minConfidence || 0) && j !== 61) {
|
||||
const cx = (0.5 + Math.trunc(i % baseSize)) / baseSize; // center.x normalized to range 0..1
|
||||
|
|
@ -92,8 +92,8 @@ async function process(res: Tensor[], outputShape: [number, number], config: Con
|
|||
const nmsScores = results.map((a) => a.score);
|
||||
let nmsIdx: number[] = [];
|
||||
if (nmsBoxes && nmsBoxes.length > 0) {
|
||||
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected, config.object.iouThreshold, config.object.minConfidence);
|
||||
nmsIdx = await nms.data();
|
||||
const nms = await tf.image.nonMaxSuppressionAsync(nmsBoxes, nmsScores, config.object.maxDetected || 0, config.object.iouThreshold, config.object.minConfidence);
|
||||
nmsIdx = Array.from(await nms.data());
|
||||
tf.dispose(nms);
|
||||
}
|
||||
|
||||
|
|
@ -105,7 +105,7 @@ async function process(res: Tensor[], outputShape: [number, number], config: Con
|
|||
return results;
|
||||
}
|
||||
|
||||
export async function predict(image: Tensor, config: Config): Promise<ObjectResult[]> {
|
||||
export async function predict(image: Tensor4D, config: Config): Promise<ObjectResult[]> {
|
||||
if (!model?.['executor']) return [];
|
||||
const skipTime = (config.object.skipTime || 0) > (now() - lastTime);
|
||||
const skipFrame = skipped < (config.object.skipFrames || 0);
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@
|
|||
* - [**MediaPipe Meet**](https://drive.google.com/file/d/1lnP1bRi9CSqQQXUHa13159vLELYDgDu0/preview)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -21,21 +21,21 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<Tensor | null> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<Tensor | null> {
|
||||
if (!model) model = await load(config);
|
||||
if (!model?.['executor'] || !model?.inputs?.[0].shape) return null; // something is wrong with the model
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.resize = tf.image.resizeBilinear(input, [model.inputs[0].shape ? model.inputs[0].shape[1] : 0, model.inputs[0].shape ? model.inputs[0].shape[2] : 0], false);
|
||||
t.norm = tf.div(t.resize, constants.tf255);
|
||||
t.res = model.execute(t.norm) as Tensor;
|
||||
t.squeeze = tf.squeeze(t.res, 0);
|
||||
t.squeeze = tf.squeeze(t.res, [0]);
|
||||
// t.softmax = tf.softmax(t.squeeze); // model meet has two channels for fg and bg
|
||||
[t.bgRaw, t.fgRaw] = tf.unstack(t.squeeze, 2);
|
||||
// t.bg = tf.softmax(t.bgRaw); // we can ignore bg channel
|
||||
t.fg = tf.softmax(t.fgRaw);
|
||||
t.mul = tf.mul(t.fg, constants.tf255);
|
||||
t.expand = tf.expandDims(t.mul, 2);
|
||||
t.output = tf.image.resizeBilinear(t.expand, [input.shape[1], input.shape[2]]);
|
||||
t.output = tf.image.resizeBilinear(t.expand as Tensor4D, [input.shape[1] || 0, input.shape[2] || 0]);
|
||||
let rgba: Tensor;
|
||||
switch (config.segmentation.mode || 'default') {
|
||||
case 'default':
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@
|
|||
* - [**Robust Video Matting**](https://github.com/PeterL1n/RobustVideoMatting)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -37,11 +37,11 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
const normalize = (r: Tensor) => tf.tidy(() => {
|
||||
const normalize = (r: Tensor): Tensor => tf.tidy(() => {
|
||||
const squeeze = tf.squeeze(r, ([0]));
|
||||
const mul = tf.mul(squeeze, constants.tf255);
|
||||
const cast = tf.cast(mul, 'int32');
|
||||
return cast as Tensor;
|
||||
return cast;
|
||||
});
|
||||
|
||||
function getRGBA(fgr: Tensor | null, pha: Tensor | null): Tensor { // gets rgba // either fgr or pha must be present
|
||||
|
|
@ -68,13 +68,13 @@ function getState(state: Tensor): Tensor { // gets internal recurrent states
|
|||
r.add = tf.add(r.expand, 1);
|
||||
r.mul = tf.mul(r.add, 127.5);
|
||||
r.cast = tf.cast(r.mul, 'int32');
|
||||
r.tile = tf.tile(r.cast, [1, 1, 3]) as Tensor;
|
||||
r.alpha = tf.fill([r.tile.shape[0] || 0, r.tile.shape[1] || 0, 1], 255, 'int32');
|
||||
return tf.concat([r.tile, r.alpha], -1) as Tensor;
|
||||
r.tile = tf.tile(r.cast, [1, 1, 3]);
|
||||
r.alpha = tf.fill([(r.tile as Tensor).shape[0] || 0, (r.tile as Tensor).shape[1] || 0, 1], 255, 'int32'); // eslint-disable-line @typescript-eslint/no-unnecessary-type-assertion
|
||||
return tf.concat([r.tile, r.alpha], -1);
|
||||
});
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<Tensor | null> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<Tensor | null> {
|
||||
if (!model) model = await load(config);
|
||||
if (!model?.['executor']) return null;
|
||||
// const expand = tf.expandDims(input, 0);
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@
|
|||
* - [**MediaPipe Selfie**](https://drive.google.com/file/d/1dCfozqknMa068vVsO2j_1FgZkW_e3VWv/preview)
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import { loadModel } from '../tfjs/load';
|
||||
import { constants } from '../tfjs/constants';
|
||||
import type { GraphModel, Tensor } from '../tfjs/types';
|
||||
import type { GraphModel, Tensor, Tensor4D } from '../tfjs/types';
|
||||
import type { Config } from '../config';
|
||||
import { env } from '../util/env';
|
||||
|
||||
|
|
@ -21,15 +21,15 @@ export async function load(config: Config): Promise<GraphModel> {
|
|||
return model;
|
||||
}
|
||||
|
||||
export async function predict(input: Tensor, config: Config): Promise<Tensor | null> {
|
||||
export async function predict(input: Tensor4D, config: Config): Promise<Tensor | null> {
|
||||
if (!model) model = await load(config);
|
||||
if (!model?.['executor'] || !model?.inputs?.[0].shape) return null; // something is wrong with the model
|
||||
const t: Record<string, Tensor> = {};
|
||||
t.resize = tf.image.resizeBilinear(input, [model.inputs[0].shape ? model.inputs[0].shape[1] : 0, model.inputs[0].shape ? model.inputs[0].shape[2] : 0], false);
|
||||
t.norm = tf.div(t.resize, constants.tf255);
|
||||
t.res = model.execute(t.norm) as Tensor;
|
||||
t.squeeze = tf.squeeze(t.res, 0); // meet.shape:[1,256,256,1], selfie.shape:[1,144,256,2]
|
||||
t.alpha = tf.image.resizeBilinear(t.squeeze, [input.shape[1], input.shape[2]]); // model selfie has a single channel that we can use directly
|
||||
t.squeeze = tf.squeeze(t.res, [0]); // meet.shape:[1,256,256,1], selfie.shape:[1,144,256,2]
|
||||
t.alpha = tf.image.resizeBilinear(t.squeeze as Tensor4D, [input.shape[1] || 0, input.shape[2] || 0]); // model selfie has a single channel that we can use directly
|
||||
t.mul = tf.mul(t.alpha, constants.tf255);
|
||||
let rgba: Tensor;
|
||||
switch (config.segmentation.mode || 'default') {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
/** TFJS backend initialization and customization */
|
||||
|
||||
import type { Human, Config } from '../human';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Human, Config, BackendEnum } from '../human';
|
||||
import { log, now } from '../util/util';
|
||||
import { env } from '../util/env';
|
||||
import * as humangl from './humangl';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as constants from './constants';
|
||||
import type { TensorInfo } from './types';
|
||||
|
||||
function registerCustomOps(config: Config) {
|
||||
const newKernels: string[] = [];
|
||||
|
|
@ -23,7 +24,7 @@ function registerCustomOps(config: Config) {
|
|||
const kernelFloorMod = {
|
||||
kernelName: 'FloorMod',
|
||||
backendName: tf.getBackend(),
|
||||
kernelFunc: (op) => tf.tidy(() => tf.add(tf.mul(tf.floorDiv(op.inputs.a / op.inputs.b), op.inputs.b), tf.mod(op.inputs.a, op.inputs.b))),
|
||||
kernelFunc: (op) => tf.tidy(() => tf.add(tf.mul(tf.floorDiv(op.inputs.a, op.inputs.b), op.inputs.b), tf.mod(op.inputs.a, op.inputs.b))),
|
||||
};
|
||||
tf.registerKernel(kernelFloorMod);
|
||||
env.kernels.push('floormod');
|
||||
|
|
@ -55,9 +56,9 @@ function registerCustomOps(config: Config) {
|
|||
backendName: tf.getBackend(),
|
||||
kernelFunc: (op) => tf.tidy(() => {
|
||||
const backend = tf.getBackend();
|
||||
tf.setBackend('cpu');
|
||||
tf.setBackend('cpu'); // eslint-disable-line @typescript-eslint/no-floating-promises
|
||||
const t = tf.image.rotateWithOffset(op.inputs.image, op.attrs.radians, op.attrs.fillValue, op.attrs.center);
|
||||
tf.setBackend(backend);
|
||||
tf.setBackend(backend); // eslint-disable-line @typescript-eslint/no-floating-promises
|
||||
return t;
|
||||
}),
|
||||
};
|
||||
|
|
@ -105,7 +106,7 @@ export async function check(instance: Human, force = false) {
|
|||
instance.config.backend = 'webgl';
|
||||
} else {
|
||||
// @ts-ignore requestAdapterInfo is not in tslib
|
||||
const adapterInfo = 'requestAdapterInfo' in adapter ? await (adapter as GPUAdapter).requestAdapterInfo() : undefined;
|
||||
const adapterInfo = 'requestAdapterInfo' in adapter ? await adapter.requestAdapterInfo() : undefined;
|
||||
// if (adapter.features) adapter.features.forEach((feature) => log('webgpu features:', feature));
|
||||
log('webgpu adapter info:', adapterInfo);
|
||||
}
|
||||
|
|
@ -130,6 +131,7 @@ export async function check(instance: Human, force = false) {
|
|||
|
||||
// customize wasm
|
||||
if (instance.config.backend === 'wasm') {
|
||||
// @ts-ignore private property
|
||||
if (tf.env().flagRegistry.CANVAS2D_WILL_READ_FREQUENTLY) tf.env().set('CANVAS2D_WILL_READ_FREQUENTLY', true);
|
||||
if (instance.config.debug) log('wasm path:', instance.config.wasmPath);
|
||||
if (typeof tf.setWasmPaths !== 'undefined') tf.setWasmPaths(instance.config.wasmPath, instance.config.wasmPlatformFetch);
|
||||
|
|
@ -137,8 +139,8 @@ export async function check(instance: Human, force = false) {
|
|||
let mt = false;
|
||||
let simd = false;
|
||||
try {
|
||||
mt = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT');
|
||||
simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT');
|
||||
mt = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT') as boolean;
|
||||
simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT') as boolean;
|
||||
if (instance.config.debug) log(`wasm execution: ${simd ? 'simd' : 'no simd'} ${mt ? 'multithreaded' : 'singlethreaded'}`);
|
||||
if (instance.config.debug && !simd) log('warning: wasm simd support is not enabled');
|
||||
} catch {
|
||||
|
|
@ -153,12 +155,15 @@ export async function check(instance: Human, force = false) {
|
|||
log('error: cannot set backend:', instance.config.backend, err);
|
||||
return false;
|
||||
}
|
||||
// @ts-ignore private property
|
||||
if (instance.config.debug) defaultFlags = JSON.parse(JSON.stringify(tf.env().flags));
|
||||
}
|
||||
|
||||
// customize humangl
|
||||
if (tf.getBackend() === 'humangl' || tf.getBackend() === 'webgl') {
|
||||
// @ts-ignore private property
|
||||
if (tf.env().flagRegistry.WEBGL_USE_SHAPES_UNIFORMS) tf.env().set('WEBGL_USE_SHAPES_UNIFORMS', true); // default=false <https://github.com/tensorflow/tfjs/issues/5205>
|
||||
// @ts-ignore private property
|
||||
if (tf.env().flagRegistry.WEBGL_EXP_CONV) tf.env().set('WEBGL_EXP_CONV', true); // default=false <https://github.com/tensorflow/tfjs/issues/6678>
|
||||
// if (tf.env().flagRegistry['WEBGL_PACK_DEPTHWISECONV']) tf.env().set('WEBGL_PACK_DEPTHWISECONV', false); // default=true <https://github.com/tensorflow/tfjs/pull/4909>
|
||||
// if (tf.env().flagRegistry.USE_SETTIMEOUTCUSTOM) tf.env().set('USE_SETTIMEOUTCUSTOM', true); // default=false <https://github.com/tensorflow/tfjs/issues/6687>
|
||||
|
|
@ -178,6 +183,7 @@ export async function check(instance: Human, force = false) {
|
|||
}
|
||||
|
||||
if (instance.config.debug) {
|
||||
// @ts-ignore private property
|
||||
const newFlags = tf.env().flags;
|
||||
const updatedFlags = {};
|
||||
for (const key of Object.keys(newFlags)) {
|
||||
|
|
@ -190,14 +196,14 @@ export async function check(instance: Human, force = false) {
|
|||
if (instance.config.flags && Object.keys(instance.config.flags).length > 0) {
|
||||
if (instance.config.debug) log('flags:', instance.config['flags']);
|
||||
for (const [key, val] of Object.entries(instance.config.flags)) {
|
||||
tf.env().set(key, val);
|
||||
tf.env().set(key, val as number | boolean);
|
||||
}
|
||||
}
|
||||
|
||||
tf.enableProdMode();
|
||||
constants.init();
|
||||
instance.performance.initBackend = Math.trunc(now() - timeStamp);
|
||||
instance.config.backend = tf.getBackend();
|
||||
instance.config.backend = tf.getBackend() as BackendEnum;
|
||||
await env.updateBackend(); // update env on backend init
|
||||
registerCustomOps(instance.config);
|
||||
// await env.updateBackend(); // update env on backend init
|
||||
|
|
@ -213,11 +219,14 @@ export function fakeOps(kernelNames: string[], config) {
|
|||
const kernelConfig = {
|
||||
kernelName,
|
||||
backendName: config.backend,
|
||||
kernelFunc: () => { if (config.debug) log('kernelFunc', kernelName, config.backend); },
|
||||
kernelFunc: (param): TensorInfo => {
|
||||
if (config.debug) log('kernelFunc', kernelName, config.backend, param);
|
||||
return param?.inputs?.info as TensorInfo;
|
||||
},
|
||||
// setupFunc: () => { if (config.debug) log('kernelFunc', kernelName, config.backend); },
|
||||
// disposeFunc: () => { if (config.debug) log('kernelFunc', kernelName, config.backend); },
|
||||
};
|
||||
tf.registerKernel(kernelConfig);
|
||||
}
|
||||
env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => (kernel.kernelName as string).toLowerCase()); // re-scan registered ops
|
||||
env.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase()); // re-scan registered ops
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Tensor } from './types';
|
||||
|
||||
export const constants: Record<string, Tensor | number | number[]> = {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
/** TFJS custom backend registration */
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import type { Human } from '../human';
|
||||
import { log } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as image from '../image/image';
|
||||
import * as models from '../models';
|
||||
import type { AnyCanvas } from '../exports';
|
||||
import type { MathBackendWebGL } from './types';
|
||||
// import { env } from '../env';
|
||||
|
||||
export const config = {
|
||||
|
|
@ -120,13 +121,15 @@ export function register(instance: Human): void {
|
|||
return;
|
||||
}
|
||||
try {
|
||||
// @ts-ignore private property
|
||||
if (tf.env().flagRegistry.WEBGL_VERSION) tf.env().set('WEBGL_VERSION', 2);
|
||||
} catch (err) {
|
||||
log('humangl error: cannot set WebGL backend flags:', err);
|
||||
return;
|
||||
}
|
||||
extensions();
|
||||
const current = tf.backend().getGPGPUContext ? tf.backend().getGPGPUContext().gl : null;
|
||||
const backend = tf.backend() as MathBackendWebGL;
|
||||
const current = typeof backend['gpgpu'] !== 'undefined' ? backend.getGPGPUContext().gl : null;
|
||||
if (current) {
|
||||
if (instance.config.debug) log('humangl backend registered:', { webgl: current.getParameter(current.VERSION) as string, renderer: current.getParameter(current.RENDERER) as string });
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, join } from '../util/util';
|
||||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import type { GraphModel } from './types';
|
||||
import type { Config } from '../config';
|
||||
import * as modelsDefs from '../../models/models.json';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,391 @@
|
|||
/* eslint-disable no-use-before-define */
|
||||
// Type definitions for long.js 4.0.0
|
||||
// Project: https://github.com/dcodeIO/long.js
|
||||
// Definitions by: Peter Kooijmans <https://github.com/peterkooijmans>
|
||||
// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
|
||||
// Definitions by: Denis Cappellin <https://github.com/cappellin>
|
||||
|
||||
export = Long;
|
||||
export as namespace Long;
|
||||
|
||||
declare const Long: Long.LongConstructor;
|
||||
type Long = Long.Long;
|
||||
declare namespace Long {
|
||||
interface LongConstructor {
|
||||
/**
|
||||
* Constructs a 64 bit two's-complement integer, given its low and high 32 bit values as signed integers. See the from* functions below for more convenient ways of constructing Longs.
|
||||
*/
|
||||
new(low: number, high?: number, unsigned?: boolean): Long;
|
||||
prototype: Long;
|
||||
/**
|
||||
* Maximum unsigned value.
|
||||
*/
|
||||
MAX_UNSIGNED_VALUE: Long;
|
||||
|
||||
/**
|
||||
* Maximum signed value.
|
||||
*/
|
||||
MAX_VALUE: Long;
|
||||
|
||||
/**
|
||||
* Minimum signed value.
|
||||
*/
|
||||
MIN_VALUE: Long;
|
||||
|
||||
/**
|
||||
* Signed negative one.
|
||||
*/
|
||||
NEG_ONE: Long;
|
||||
|
||||
/**
|
||||
* Signed one.
|
||||
*/
|
||||
ONE: Long;
|
||||
|
||||
/**
|
||||
* Unsigned one.
|
||||
*/
|
||||
UONE: Long;
|
||||
|
||||
/**
|
||||
* Unsigned zero.
|
||||
*/
|
||||
UZERO: Long;
|
||||
|
||||
/**
|
||||
* Signed zero
|
||||
*/
|
||||
ZERO: Long;
|
||||
|
||||
/**
|
||||
* Returns a Long representing the 64 bit integer that comes by concatenating the given low and high bits. Each is assumed to use 32 bits.
|
||||
*/
|
||||
fromBits(lowBits:number, highBits:number, unsigned?:boolean): Long;
|
||||
|
||||
/**
|
||||
* Returns a Long representing the given 32 bit integer value.
|
||||
*/
|
||||
fromInt(value: number, unsigned?: boolean): Long;
|
||||
|
||||
/**
|
||||
* Returns a Long representing the given value, provided that it is a finite number. Otherwise, zero is returned.
|
||||
*/
|
||||
fromNumber(value: number, unsigned?: boolean): Long;
|
||||
|
||||
/**
|
||||
* Returns a Long representation of the given string, written using the specified radix.
|
||||
*/
|
||||
fromString(str: string, unsigned?: boolean | number, radix?: number): Long;
|
||||
|
||||
/**
|
||||
* Creates a Long from its byte representation.
|
||||
*/
|
||||
fromBytes(bytes: number[], unsigned?: boolean, le?: boolean): Long;
|
||||
|
||||
/**
|
||||
* Creates a Long from its little endian byte representation.
|
||||
*/
|
||||
fromBytesLE(bytes: number[], unsigned?: boolean): Long;
|
||||
|
||||
/**
|
||||
* Creates a Long from its little endian byte representation.
|
||||
*/
|
||||
fromBytesBE(bytes: number[], unsigned?: boolean): Long;
|
||||
|
||||
/**
|
||||
* Tests if the specified object is a Long.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
isLong(obj: any): obj is Long;
|
||||
|
||||
/**
|
||||
* Converts the specified value to a Long.
|
||||
*/
|
||||
fromValue(val: Long | number | string | {low: number, high: number, unsigned: boolean}): Long;
|
||||
}
|
||||
interface Long
|
||||
{
|
||||
/**
|
||||
* The high 32 bits as a signed value.
|
||||
*/
|
||||
high: number;
|
||||
|
||||
/**
|
||||
* The low 32 bits as a signed value.
|
||||
*/
|
||||
low: number;
|
||||
|
||||
/**
|
||||
* Whether unsigned or not.
|
||||
*/
|
||||
unsigned: boolean;
|
||||
|
||||
/**
|
||||
* Returns the sum of this and the specified Long.
|
||||
*/
|
||||
add(addend: number | Long | string): Long;
|
||||
|
||||
/**
|
||||
* Returns the bitwise AND of this Long and the specified.
|
||||
*/
|
||||
and(other: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Compares this Long's value with the specified's.
|
||||
*/
|
||||
compare(other: Long | number | string): number;
|
||||
|
||||
/**
|
||||
* Compares this Long's value with the specified's.
|
||||
*/
|
||||
comp(other: Long | number | string): number;
|
||||
|
||||
/**
|
||||
* Returns this Long divided by the specified.
|
||||
*/
|
||||
divide(divisor: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long divided by the specified.
|
||||
*/
|
||||
div(divisor: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value equals the specified's.
|
||||
*/
|
||||
equals(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value equals the specified's.
|
||||
*/
|
||||
eq(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Gets the high 32 bits as a signed integer.
|
||||
*/
|
||||
getHighBits(): number;
|
||||
|
||||
/**
|
||||
* Gets the high 32 bits as an unsigned integer.
|
||||
*/
|
||||
getHighBitsUnsigned(): number;
|
||||
|
||||
/**
|
||||
* Gets the low 32 bits as a signed integer.
|
||||
*/
|
||||
getLowBits(): number;
|
||||
|
||||
/**
|
||||
* Gets the low 32 bits as an unsigned integer.
|
||||
*/
|
||||
getLowBitsUnsigned(): number;
|
||||
|
||||
/**
|
||||
* Gets the number of bits needed to represent the absolute value of this Long.
|
||||
*/
|
||||
getNumBitsAbs(): number;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is greater than the specified's.
|
||||
*/
|
||||
greaterThan(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is greater than the specified's.
|
||||
*/
|
||||
gt(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is greater than or equal the specified's.
|
||||
*/
|
||||
greaterThanOrEqual(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is greater than or equal the specified's.
|
||||
*/
|
||||
gte(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is even.
|
||||
*/
|
||||
isEven(): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is negative.
|
||||
*/
|
||||
isNegative(): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is odd.
|
||||
*/
|
||||
isOdd(): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is positive.
|
||||
*/
|
||||
isPositive(): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value equals zero.
|
||||
*/
|
||||
isZero(): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is less than the specified's.
|
||||
*/
|
||||
lessThan(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is less than the specified's.
|
||||
*/
|
||||
lt(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is less than or equal the specified's.
|
||||
*/
|
||||
lessThanOrEqual(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value is less than or equal the specified's.
|
||||
*/
|
||||
lte(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Returns this Long modulo the specified.
|
||||
*/
|
||||
modulo(other: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long modulo the specified.
|
||||
*/
|
||||
mod(other: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Returns the product of this and the specified Long.
|
||||
*/
|
||||
multiply(multiplier: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Returns the product of this and the specified Long.
|
||||
*/
|
||||
mul(multiplier: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Negates this Long's value.
|
||||
*/
|
||||
negate(): Long;
|
||||
|
||||
/**
|
||||
* Negates this Long's value.
|
||||
*/
|
||||
neg(): Long;
|
||||
|
||||
/**
|
||||
* Returns the bitwise NOT of this Long.
|
||||
*/
|
||||
not(): Long;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value differs from the specified's.
|
||||
*/
|
||||
notEquals(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Tests if this Long's value differs from the specified's.
|
||||
*/
|
||||
neq(other: Long | number | string): boolean;
|
||||
|
||||
/**
|
||||
* Returns the bitwise OR of this Long and the specified.
|
||||
*/
|
||||
or(other: Long | number | string): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits shifted to the left by the given amount.
|
||||
*/
|
||||
shiftLeft(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits shifted to the left by the given amount.
|
||||
*/
|
||||
shl(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits arithmetically shifted to the right by the given amount.
|
||||
*/
|
||||
shiftRight(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits arithmetically shifted to the right by the given amount.
|
||||
*/
|
||||
shr(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits logically shifted to the right by the given amount.
|
||||
*/
|
||||
shiftRightUnsigned(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns this Long with bits logically shifted to the right by the given amount.
|
||||
*/
|
||||
shru(numBits: number | Long): Long;
|
||||
|
||||
/**
|
||||
* Returns the difference of this and the specified Long.
|
||||
*/
|
||||
subtract(subtrahend: number | Long | string): Long;
|
||||
|
||||
/**
|
||||
* Returns the difference of this and the specified Long.
|
||||
*/
|
||||
sub(subtrahend: number | Long |string): Long;
|
||||
|
||||
/**
|
||||
* Converts the Long to a 32 bit integer, assuming it is a 32 bit integer.
|
||||
*/
|
||||
toInt(): number;
|
||||
|
||||
/**
|
||||
* Converts the Long to a the nearest floating-point representation of this value (double, 53 bit mantissa).
|
||||
*/
|
||||
toNumber(): number;
|
||||
|
||||
/**
|
||||
* Converts this Long to its byte representation.
|
||||
*/
|
||||
|
||||
toBytes(le?: boolean): number[];
|
||||
|
||||
/**
|
||||
* Converts this Long to its little endian byte representation.
|
||||
*/
|
||||
|
||||
toBytesLE(): number[];
|
||||
|
||||
/**
|
||||
* Converts this Long to its big endian byte representation.
|
||||
*/
|
||||
|
||||
toBytesBE(): number[];
|
||||
|
||||
/**
|
||||
* Converts this Long to signed.
|
||||
*/
|
||||
toSigned(): Long;
|
||||
|
||||
/**
|
||||
* Converts the Long to a string written in the specified radix.
|
||||
*/
|
||||
toString(radix?: number): string;
|
||||
|
||||
/**
|
||||
* Converts this Long to unsigned.
|
||||
*/
|
||||
toUnsigned(): Long;
|
||||
|
||||
/**
|
||||
* Returns the bitwise XOR of this Long and the given one.
|
||||
*/
|
||||
xor(other: Long | number | string): Long;
|
||||
}
|
||||
}
|
||||
|
|
@ -12,9 +12,8 @@ export declare const version: {
|
|||
|
||||
export * from '@tensorflow/tfjs-core';
|
||||
export * from '@tensorflow/tfjs-converter';
|
||||
export * from '@tensorflow/tfjs-data';
|
||||
export * from '@tensorflow/tfjs-layers';
|
||||
export * from '@tensorflow/tfjs-backend-cpu';
|
||||
// export * from '@tensorflow/tfjs-data';
|
||||
// export * from '@tensorflow/tfjs-layers';
|
||||
// export * from '@tensorflow/tfjs-backend-cpu';
|
||||
export * from '@tensorflow/tfjs-backend-wasm';
|
||||
export * from '@tensorflow/tfjs-backend-webgl';
|
||||
export {};
|
||||
|
|
|
|||
|
|
@ -1,18 +1,25 @@
|
|||
/** TFJS common types exports */
|
||||
/* eslint-disable import/no-extraneous-dependencies */
|
||||
|
||||
/**
|
||||
* TensorFlow Tensor type
|
||||
* TensorFlow
|
||||
* @external
|
||||
*/
|
||||
export type { Tensor, TensorLike, Rank } from '@tensorflow/tfjs-core/dist/index'; // eslint-disable-line import/no-extraneous-dependencies
|
||||
export type { version_core } from '@tensorflow/tfjs-core'; // eslint-disable-line camelcase
|
||||
export type { image, browser, io } from '@tensorflow/tfjs-core';
|
||||
export type { clone, sum, transpose, addN, softmax, tile, cast, unstack, pad, div, split, squeeze, add, mul, sub, stack, sigmoid, argMax, reshape, max, mod, min, floorDiv } from '@tensorflow/tfjs-core';
|
||||
export type { slice, slice3d, slice4d, concat, concat2d, expandDims } from '@tensorflow/tfjs-core';
|
||||
export type { fill, scalar, tensor2d, zeros, tensor, dispose, tensor1d, tidy, ready, getBackend, registerKernel, engine, env, setBackend, enableProdMode, getKernelsForBackend, findBackend, registerBackend, backend } from '@tensorflow/tfjs-core';
|
||||
export type { Tensor, Tensor1D, Tensor2D, Tensor3D, Tensor4D, TensorLike, TensorInfo, DataType, Rank, TensorContainer, TensorContainerObject } from '@tensorflow/tfjs-core';
|
||||
export type { loadGraphModel, GraphModel } from '@tensorflow/tfjs-converter';
|
||||
export type { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
|
||||
export type { setWebGLContext, GPGPUContext, MathBackendWebGL } from '@tensorflow/tfjs-backend-webgl';
|
||||
|
||||
/**
|
||||
* TensorFlow GraphModel type
|
||||
* @external
|
||||
*/
|
||||
export type { GraphModel } from '@tensorflow/tfjs-converter/dist/index'; // eslint-disable-line import/no-extraneous-dependencies
|
||||
|
||||
/** Tensorflow Long type
|
||||
* @external long
|
||||
*/
|
||||
// export type { Long } from 'long';
|
||||
export declare const version: {
|
||||
'tfjs-core': string;
|
||||
'tfjs-backend-cpu': string;
|
||||
'tfjs-backend-webgl': string;
|
||||
'tfjs-data': string;
|
||||
'tfjs-layers': string;
|
||||
'tfjs-converter': string;
|
||||
tfjs: string;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import * as tf from '../../dist/tfjs.esm.js';
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import * as image from '../image/image';
|
||||
import type { MathBackendWebGL } from '../tfjs/types';
|
||||
|
||||
/** Env class that holds detected capabilities */
|
||||
export class Env {
|
||||
|
|
@ -124,14 +125,14 @@ export class Env {
|
|||
// analyze backends
|
||||
this.backends = Object.keys(tf.engine().registryFactory);
|
||||
this.tensorflow = {
|
||||
version: (tf.backend().binding ? tf.backend().binding.TF_Version : undefined),
|
||||
gpu: (tf.backend().binding ? tf.backend().binding.isUsingGpuDevice() : undefined),
|
||||
version: (tf.backend()['binding'] ? tf.backend()['binding'].TF_Version : undefined),
|
||||
gpu: (tf.backend()['binding'] ? tf.backend()['binding'].isUsingGpuDevice() : undefined),
|
||||
};
|
||||
this.wasm.supported = typeof WebAssembly !== 'undefined';
|
||||
this.wasm.backend = this.backends.includes('wasm');
|
||||
if (this.wasm.supported && this.wasm.backend && tf.getBackend() === 'wasm') {
|
||||
this.wasm.simd = tf.env().get('WASM_HAS_SIMD_SUPPORT');
|
||||
this.wasm.multithread = tf.env().get('WASM_HAS_MULTITHREAD_SUPPORT');
|
||||
this.wasm.simd = tf.env().get('WASM_HAS_SIMD_SUPPORT') as boolean;
|
||||
this.wasm.multithread = tf.env().get('WASM_HAS_MULTITHREAD_SUPPORT') as boolean;
|
||||
}
|
||||
const c = image.canvas(100, 100);
|
||||
const ctx = c ? c.getContext('webgl2') : undefined; // causes too many gl contexts
|
||||
|
|
@ -139,7 +140,8 @@ export class Env {
|
|||
this.webgl.supported = typeof ctx !== 'undefined';
|
||||
this.webgl.backend = this.backends.includes('webgl');
|
||||
if (this.webgl.supported && this.webgl.backend && (tf.getBackend() === 'webgl' || tf.getBackend() === 'humangl')) {
|
||||
const gl = tf.backend().gpgpu !== 'undefined' ? await tf.backend().getGPGPUContext().gl : null;
|
||||
const backend = tf.backend() as MathBackendWebGL;
|
||||
const gl = typeof backend['gpgpu'] !== 'undefined' ? backend.getGPGPUContext().gl : null;
|
||||
if (gl) {
|
||||
this.webgl.version = gl.getParameter(gl.VERSION);
|
||||
this.webgl.renderer = gl.getParameter(gl.RENDERER);
|
||||
|
|
@ -156,7 +158,7 @@ export class Env {
|
|||
this.webgpu.supported = false;
|
||||
}
|
||||
try {
|
||||
this.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => (kernel.kernelName as string).toLowerCase());
|
||||
this.kernels = tf.getKernelsForBackend(tf.getBackend()).map((kernel) => kernel.kernelName.toLowerCase());
|
||||
} catch { /**/ }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,16 @@
|
|||
* Warmup algorithm that uses embedded images to exercise loaded models for faster future inference
|
||||
*/
|
||||
|
||||
import * as tf from 'dist/tfjs.esm.js';
|
||||
import { log, now, mergeDeep } from './util/util';
|
||||
import * as sample from './sample';
|
||||
import * as tf from '../dist/tfjs.esm.js';
|
||||
import * as image from './image/image';
|
||||
import * as backend from './tfjs/backend';
|
||||
import { env } from './util/env';
|
||||
import type { Config } from './config';
|
||||
import type { Result } from './result';
|
||||
import { Human, models } from './human';
|
||||
import type { Tensor } from './exports';
|
||||
import type { Tensor, DataType, MathBackendWebGL } from './tfjs/types';
|
||||
|
||||
async function warmupBitmap(instance: Human): Promise<Result | undefined> {
|
||||
const b64toBlob = (base64: string, type = 'application/octet-stream') => fetch(`data:${type};base64,${base64}`).then((res) => res.blob());
|
||||
|
|
@ -110,10 +110,11 @@ async function runInference(instance: Human) {
|
|||
|
||||
/** Runs pre-compile on all loaded models */
|
||||
export async function runCompile(instance: Human) {
|
||||
// @ts-ignore private property
|
||||
if (!tf.env().flagRegistry.ENGINE_COMPILE_ONLY) return; // tfjs does not support compile-only inference
|
||||
const backendType = tf.getBackend();
|
||||
const webGLBackend = tf.backend();
|
||||
if ((backendType !== 'webgl' && backendType !== 'humangl') || !webGLBackend?.checkCompileCompletion) {
|
||||
const webGLBackend = tf.backend() as MathBackendWebGL;
|
||||
if ((backendType !== 'webgl' && backendType !== 'humangl') || !webGLBackend?.['checkCompileCompletion']) {
|
||||
// log('compile pass: skip');
|
||||
return;
|
||||
}
|
||||
|
|
@ -122,7 +123,7 @@ export async function runCompile(instance: Human) {
|
|||
const compiledModels: string[] = [];
|
||||
for (const [modelName, model] of Object.entries(instance.models).filter(([key, val]) => (key !== null && val !== null))) {
|
||||
const shape = (model.inputs?.[0]?.shape) ? [...model.inputs[0].shape] : [1, 64, 64, 3];
|
||||
const dtype: string = (model.inputs?.[0]?.dtype) ? model.inputs[0].dtype : 'float32';
|
||||
const dtype: DataType = (model.inputs?.[0]?.dtype) ? model.inputs[0].dtype : 'float32';
|
||||
for (let dim = 0; dim < shape.length; dim++) {
|
||||
if (shape[dim] === -1) shape[dim] = dim === 0 ? 1 : 64; // override batch number and any dynamic dimensions
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,40 @@
|
|||
2022-10-13 09:27:05 [32mDATA: [39m Build {"name":"@vladmandic/human","version":"2.11.1"}
|
||||
2022-10-13 09:27:05 [36mINFO: [39m Application: {"name":"@vladmandic/human","version":"2.11.1"}
|
||||
2022-10-13 09:27:05 [36mINFO: [39m Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
|
||||
2022-10-13 09:27:05 [36mINFO: [39m Toolchain: {"build":"0.7.14","esbuild":"0.15.10","typescript":"4.8.4","typedoc":"0.23.16","eslint":"8.25.0"}
|
||||
2022-10-13 09:27:05 [36mINFO: [39m Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":78,"inputBytes":669029,"outputBytes":315176}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":78,"inputBytes":669033,"outputBytes":315180}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":78,"inputBytes":669085,"outputBytes":315230}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":358}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1088,"outputBytes":583}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":78,"inputBytes":669004,"outputBytes":313912}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1265,"outputBytes":2814441}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":78,"inputBytes":3482862,"outputBytes":1693611}
|
||||
2022-10-13 09:27:05 [35mSTATE:[39m Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":78,"inputBytes":3482862,"outputBytes":3111620}
|
||||
2022-10-13 09:27:10 [35mSTATE:[39m Typings: {"input":"src/human.ts","output":"types/lib","files":15}
|
||||
2022-10-13 09:27:12 [35mSTATE:[39m TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true}
|
||||
2022-10-13 09:27:12 [35mSTATE:[39m Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5684,"outputBytes":2632}
|
||||
2022-10-13 09:27:12 [35mSTATE:[39m Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17155,"outputBytes":9175}
|
||||
2022-10-13 09:27:23 [35mSTATE:[39m Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":114,"errors":0,"warnings":0}
|
||||
2022-10-13 09:27:23 [35mSTATE:[39m ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
|
||||
2022-10-13 09:27:23 [35mSTATE:[39m Copy: {"input":"tfjs/tfjs.esm.d.ts"}
|
||||
2022-10-13 09:27:23 [36mINFO: [39m Done...
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m API-Extractor: {"succeeeded":true,"errors":0,"warnings":197}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Filter: {"input":"types/human.d.ts"}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Link: {"input":"types/human.d.ts"}
|
||||
2022-10-13 09:27:24 [36mINFO: [39m Analyze models: {"folders":8,"result":"models/models.json"}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"./models","models":12}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../human-models/models","models":43}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../blazepose/model/","models":4}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../anti-spoofing/model","models":1}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../efficientpose/models","models":3}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../insightface/models","models":5}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../movenet/models","models":3}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models {"folder":"../nanodet/models","models":4}
|
||||
2022-10-13 09:27:24 [35mSTATE:[39m Models: {"count":58,"totalSize":386543911}
|
||||
2022-10-13 09:27:24 [36mINFO: [39m Human Build complete... {"logFile":"test/build.log"}
|
||||
2022-10-16 20:28:16 [32mDATA: [39m Build {"name":"@vladmandic/human","version":"3.0.0"}
|
||||
2022-10-16 20:28:16 [36mINFO: [39m Application: {"name":"@vladmandic/human","version":"3.0.0"}
|
||||
2022-10-16 20:28:16 [36mINFO: [39m Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true}
|
||||
2022-10-16 20:28:16 [36mINFO: [39m Toolchain: {"build":"0.7.14","esbuild":"0.15.11","typescript":"4.8.4","typedoc":"0.23.16","eslint":"8.25.0"}
|
||||
2022-10-16 20:28:16 [36mINFO: [39m Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Clean: {"locations":["dist/*","types/*","typedoc/*"]}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":159,"outputBytes":608}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":78,"inputBytes":671035,"outputBytes":315423}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":167,"outputBytes":612}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":78,"inputBytes":671039,"outputBytes":315427}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":206,"outputBytes":664}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":78,"inputBytes":671091,"outputBytes":315477}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1125,"outputBytes":351}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1081,"outputBytes":576}
|
||||
2022-10-16 20:28:16 [35mSTATE:[39m Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":78,"inputBytes":671003,"outputBytes":314166}
|
||||
2022-10-16 20:28:17 [35mSTATE:[39m Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":11,"inputBytes":1354,"outputBytes":2868616}
|
||||
2022-10-16 20:28:17 [35mSTATE:[39m Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":78,"inputBytes":3539043,"outputBytes":1717157}
|
||||
2022-10-16 20:28:17 [35mSTATE:[39m Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":78,"inputBytes":3539043,"outputBytes":3175031}
|
||||
2022-10-16 20:28:27 [35mSTATE:[39m Typings: {"input":"src/human.ts","output":"types/lib","files":15}
|
||||
2022-10-16 20:28:29 [35mSTATE:[39m TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true}
|
||||
2022-10-16 20:28:29 [35mSTATE:[39m Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5672,"outputBytes":2632}
|
||||
2022-10-16 20:28:29 [35mSTATE:[39m Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17134,"outputBytes":9181}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":115,"errors":0,"warnings":0}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Copy: {"input":"src/tfjs/tfjs.esm.d.ts","output":"dist/tfjs.esm.d.ts"}
|
||||
2022-10-16 20:28:39 [36mINFO: [39m Done...
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m API-Extractor: {"succeeeded":true,"errors":0,"warnings":195}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Filter: {"input":"types/human.d.ts"}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Link: {"input":"types/human.d.ts"}
|
||||
2022-10-16 20:28:39 [36mINFO: [39m Analyze models: {"folders":8,"result":"models/models.json"}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"./models","models":12}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../human-models/models","models":43}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../blazepose/model/","models":4}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../anti-spoofing/model","models":1}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../efficientpose/models","models":3}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../insightface/models","models":5}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../movenet/models","models":3}
|
||||
2022-10-16 20:28:39 [35mSTATE:[39m Models {"folder":"../nanodet/models","models":4}
|
||||
2022-10-16 20:28:40 [35mSTATE:[39m Models: {"count":58,"totalSize":386543911}
|
||||
2022-10-16 20:28:40 [36mINFO: [39m Human Build complete... {"logFile":"test/build.log"}
|
||||
|
|
|
|||
1705
test/test.log
1705
test/test.log
File diff suppressed because it is too large
Load Diff
|
|
@ -54,6 +54,7 @@
|
|||
"exclude": ["node_modules/", "types/", "dist/**/*.js"],
|
||||
"include": ["src", "tfjs/*.ts", "types/human.d.ts", "test/**/*.ts", "demo/**/*.ts"],
|
||||
"typedocOptions": {
|
||||
"externalPattern": ["node_modules/", "tfjs/"]
|
||||
"excludeExternals": true,
|
||||
"externalPattern": ["**/node_modules/**", "tfjs/"]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue