diff --git a/CHANGELOG.md b/CHANGELOG.md index 4918e24b..cb445bf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ## Changelog -### **HEAD -> main** 2022/10/28 mandic00@live.com +### **HEAD -> main** 2022/11/04 mandic00@live.com - add named exports - add draw label templates diff --git a/TODO.md b/TODO.md index ca9bc1dd..f9914cb4 100644 --- a/TODO.md +++ b/TODO.md @@ -51,10 +51,29 @@ No support for running in **web workers** as Safari still does not support `Offs ## Pending Release Changes +Optimizations: +- Enabled high-resolution optimizations + Internal limits are increased from **2k** to **4k** +- Enhanced device capabilities detection + See `human.env.[agent, wasm, webgl, webgpu]` for details +- If `config.backend` is not set, Human will auto-select best backend + based on device capabilities +- Enhanced support for `webgpu` + +Features: +- Add [draw label templates](https://github.com/vladmandic/human/wiki/Draw) +- Add `config.filter.autoBrightness` (*enabled by default*) + Per-frame video on-the-fly brightness adjustments + Which significantly increases performance and precision in poorly lit scenes +- Improved `config.filter.equalization` (*disabled by default*) + Image and video on-demand histogram equalization + +Architecture: - Upgrade to TFJS 4.0 with **strong typing** see [notes](https://github.com/vladmandic/human#typedefs) on how to use - `TypeDef` refactoring -- Add [draw label templates](https://github.com/vladmandic/human/wiki/Draw) - Reduce build dependencies `Human` is now 30% smaller :) + As usual, `Human` has **zero** runtime dependencies, + all *devDependencies* are only to rebuild `Human` itself - Add named export for improved bundler support when using non-default imports diff --git a/demo/typescript/index.js b/demo/typescript/index.js index 8a067242..d3a60185 100644 --- a/demo/typescript/index.js +++ b/demo/typescript/index.js @@ -4,6 +4,6 @@ author: ' */ -import*as m from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1,flip:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0},antispoof:{enabled:!0},liveness:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},segmentation:{enabled:!1},gesture:{enabled:!0}},e=new m.Human(w);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;var a={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},s={detectFPS:0,drawFPS:0,frames:0,averageMs:0},o=(...t)=>{a.log.innerText+=t.join(" ")+` -`,console.log(...t)},d=t=>a.fps.innerText=t,v=t=>a.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(t).replace(/"|{|}/g,"").replace(/,/g," | ");async function f(){if(!a.video.paused){n.start===0&&(n.start=e.now()),await e.detect(a.video);let t=e.tf.memory().numTensors;t-n.tensors!==0&&o("allocated tensors:",t-n.tensors),n.tensors=t,s.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,s.frames++,s.averageMs=Math.round(1e3*(e.now()-n.start)/s.frames)/1e3,s.frames%100===0&&!a.video.paused&&o("performance",{...s,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(f)}async function u(){var i,l,c;if(!a.video.paused){let r=e.next(e.result);e.config.filter.flip?e.draw.canvas(r.canvas,a.canvas):e.draw.canvas(a.video,a.canvas);let p={bodyLabels:`person confidence [score] and ${(c=(l=(i=e.result)==null?void 0:i.body)==null?void 0:l[0])==null?void 0:c.keypoints.length} keypoints`};await e.draw.all(a.canvas,r,p),v(r.performance)}let t=e.now();s.drawFPS=Math.round(1e3*1e3/(t-n.draw))/1e3,n.draw=t,d(a.video.paused?"paused":`fps: ${s.detectFPS.toFixed(1).padStart(5," ")} detect | ${s.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(u,30)}async function g(){await e.webcam.start({element:a.video,crop:!0}),a.canvas.width=e.webcam.width,a.canvas.height=e.webcam.height,a.canvas.onclick=async()=>{e.webcam.paused?await e.webcam.play():e.webcam.pause()}}async function b(){o("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),o("platform:",e.env.platform,"| agent:",e.env.agent),d("loading..."),await e.load(),o("backend:",e.tf.getBackend(),"| available:",e.env.backends),o("models stats:",e.getModelStats()),o("models loaded:",Object.values(e.models).filter(t=>t!==null).length),o("environment",e.env),d("initializing..."),await e.warmup(),await g(),await f(),await u()}window.onload=b; +import*as m from"../../dist/human.esm.js";var f=1920,g={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1,flip:!1,width:f},face:{enabled:!0,detector:{rotation:!0},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0},antispoof:{enabled:!0},liveness:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!1},segmentation:{enabled:!1},gesture:{enabled:!0}},e=new m.Human(g);e.env.perfadd=!1;e.draw.options.font='small-caps 18px "Lato"';e.draw.options.lineHeight=20;var a={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},n={detect:0,draw:0,tensors:0,start:0},o={detectFPS:0,drawFPS:0,frames:0,averageMs:0},s=(...t)=>{a.log.innerText+=t.join(" ")+` +`,console.log(...t)},r=t=>a.fps.innerText=t,b=t=>a.perf.innerText="tensors:"+e.tf.memory().numTensors.toString()+" | performance: "+JSON.stringify(t).replace(/"|{|}/g,"").replace(/,/g," | ");async function u(){if(!a.video.paused){n.start===0&&(n.start=e.now()),await e.detect(a.video);let t=e.tf.memory().numTensors;t-n.tensors!==0&&s("allocated tensors:",t-n.tensors),n.tensors=t,o.detectFPS=Math.round(1e3*1e3/(e.now()-n.detect))/1e3,o.frames++,o.averageMs=Math.round(1e3*(e.now()-n.start)/o.frames)/1e3,o.frames%100===0&&!a.video.paused&&s("performance",{...o,tensors:n.tensors})}n.detect=e.now(),requestAnimationFrame(u)}async function p(){var d,i,l;if(!a.video.paused){let c=e.next(e.result),w=await e.image(a.video);e.draw.canvas(w.canvas,a.canvas);let v={bodyLabels:`person confidence [score] and ${(l=(i=(d=e.result)==null?void 0:d.body)==null?void 0:i[0])==null?void 0:l.keypoints.length} keypoints`};await e.draw.all(a.canvas,c,v),b(c.performance)}let t=e.now();o.drawFPS=Math.round(1e3*1e3/(t-n.draw))/1e3,n.draw=t,r(a.video.paused?"paused":`fps: ${o.detectFPS.toFixed(1).padStart(5," ")} detect | ${o.drawFPS.toFixed(1).padStart(5," ")} draw`),setTimeout(p,30)}async function h(){await e.webcam.start({element:a.video,crop:!0,width:f}),a.canvas.width=e.webcam.width,a.canvas.height=e.webcam.height,a.canvas.onclick=async()=>{e.webcam.paused?await e.webcam.play():e.webcam.pause()}}async function y(){s("human version:",e.version,"| tfjs version:",e.tf.version["tfjs-core"]),s("platform:",e.env.platform,"| agent:",e.env.agent),r("loading..."),await e.load(),s("backend:",e.tf.getBackend(),"| available:",e.env.backends),s("models stats:",e.getModelStats()),s("models loaded:",Object.values(e.models).filter(t=>t!==null).length),s("environment",e.env),r("initializing..."),await e.warmup(),await h(),await u(),await p()}window.onload=y; //# sourceMappingURL=index.js.map diff --git a/demo/typescript/index.js.map b/demo/typescript/index.js.map index f0d25ada..9dc875a2 100644 --- a/demo/typescript/index.js.map +++ b/demo/typescript/index.js.map @@ -1,7 +1,7 @@ { "version": 3, "sources": ["index.ts"], - "sourcesContent": ["/**\n * Human demo for browsers\n * @default Human Library\n * @summary \n * @author \n * @copyright \n * @license MIT\n */\n\nimport * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human\n\nconst humanConfig: Partial = { // user configuration for human, used to fine-tune behavior\n // backend: 'wasm',\n modelBasePath: '../../models',\n filter: { enabled: true, equalization: false, flip: false },\n face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } },\n body: { enabled: true },\n hand: { enabled: true },\n object: { enabled: false },\n segmentation: { enabled: false },\n gesture: { enabled: true },\n};\n\nconst human = new H.Human(humanConfig); // create instance of human with overrides from user configuration\n\nhuman.env.perfadd = false; // is performance data showing instant or total values\nhuman.draw.options.font = 'small-caps 18px \"Lato\"'; // set font used to draw labels when using draw methods\nhuman.draw.options.lineHeight = 20;\n// human.draw.options.fillPolygons = true;\n\nconst dom = { // grab instances of dom objects so we dont have to look them up later\n video: document.getElementById('video') as HTMLVideoElement,\n canvas: document.getElementById('canvas') as HTMLCanvasElement,\n log: document.getElementById('log') as HTMLPreElement,\n fps: document.getElementById('status') as HTMLPreElement,\n perf: document.getElementById('performance') as HTMLDivElement,\n};\nconst timestamp = { detect: 0, draw: 0, tensors: 0, start: 0 }; // holds information used to calculate performance and possible memory leaks\nconst fps = { detectFPS: 0, drawFPS: 0, frames: 0, averageMs: 0 }; // holds calculated fps information for both detect and screen refresh\n\nconst log = (...msg) => { // helper method to output messages\n dom.log.innerText += msg.join(' ') + '\\n';\n console.log(...msg); // eslint-disable-line no-console\n};\nconst status = (msg) => dom.fps.innerText = msg; // print status element\nconst perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors.toString() + ' | performance: ' + JSON.stringify(msg).replace(/\"|{|}/g, '').replace(/,/g, ' | '); // print performance element\n\nasync function detectionLoop() { // main detection loop\n if (!dom.video.paused) {\n if (timestamp.start === 0) timestamp.start = human.now();\n // log('profiling data:', await human.profile(dom.video));\n await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result\n const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks\n if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak\n timestamp.tensors = tensors;\n fps.detectFPS = Math.round(1000 * 1000 / (human.now() - timestamp.detect)) / 1000;\n fps.frames++;\n fps.averageMs = Math.round(1000 * (human.now() - timestamp.start) / fps.frames) / 1000;\n if (fps.frames % 100 === 0 && !dom.video.paused) log('performance', { ...fps, tensors: timestamp.tensors });\n }\n timestamp.detect = human.now();\n requestAnimationFrame(detectionLoop); // start new frame immediately\n}\n\nasync function drawLoop() { // main screen refresh loop\n if (!dom.video.paused) {\n const interpolated = human.next(human.result); // smoothen result using last-known results\n if (human.config.filter.flip) human.draw.canvas(interpolated.canvas as HTMLCanvasElement, dom.canvas); // draw processed image to screen canvas\n else human.draw.canvas(dom.video, dom.canvas); // draw original video to screen canvas // better than using procesed image as this loop happens faster than processing loop\n\n const opt: Partial = { bodyLabels: `person confidence [score] and ${human.result?.body?.[0]?.keypoints.length} keypoints` };\n await human.draw.all(dom.canvas, interpolated, opt); // draw labels, boxes, lines, etc.\n perf(interpolated.performance); // write performance data\n }\n const now = human.now();\n fps.drawFPS = Math.round(1000 * 1000 / (now - timestamp.draw)) / 1000;\n timestamp.draw = now;\n status(dom.video.paused ? 'paused' : `fps: ${fps.detectFPS.toFixed(1).padStart(5, ' ')} detect | ${fps.drawFPS.toFixed(1).padStart(5, ' ')} draw`); // write status\n setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps\n}\n\nasync function webCam() {\n await human.webcam.start({ element: dom.video, crop: true }); // use human webcam helper methods and associate webcam stream with a dom element\n dom.canvas.width = human.webcam.width;\n dom.canvas.height = human.webcam.height;\n dom.canvas.onclick = async () => { // pause when clicked on screen and resume on next click\n if (human.webcam.paused) await human.webcam.play();\n else human.webcam.pause();\n };\n}\n\nasync function main() { // main entry point\n log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);\n log('platform:', human.env.platform, '| agent:', human.env.agent);\n status('loading...');\n await human.load(); // preload all models\n log('backend:', human.tf.getBackend(), '| available:', human.env.backends);\n log('models stats:', human.getModelStats());\n log('models loaded:', Object.values(human.models).filter((model) => model !== null).length);\n log('environment', human.env);\n status('initializing...');\n await human.warmup(); // warmup function to initialize backend for future faster detection\n await webCam(); // start webcam\n await detectionLoop(); // start detection loop\n await drawLoop(); // start draw loop\n}\n\nwindow.onload = main;\n"], - "mappings": ";;;;;;AASA,UAAYA,MAAO,0BAEnB,IAAMC,EAAiC,CAErC,cAAe,eACf,OAAQ,CAAE,QAAS,GAAM,aAAc,GAAO,KAAM,EAAM,EAC1D,KAAM,CAAE,QAAS,GAAM,SAAU,CAAE,SAAU,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,UAAW,CAAE,QAAS,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,YAAa,CAAE,QAAS,EAAK,EAAG,QAAS,CAAE,QAAS,EAAK,EAAG,UAAW,CAAE,QAAS,EAAK,EAAG,SAAU,CAAE,QAAS,EAAK,CAAE,EAC7P,KAAM,CAAE,QAAS,EAAK,EACtB,KAAM,CAAE,QAAS,EAAK,EACtB,OAAQ,CAAE,QAAS,EAAM,EACzB,aAAc,CAAE,QAAS,EAAM,EAC/B,QAAS,CAAE,QAAS,EAAK,CAC3B,EAEMC,EAAQ,IAAM,QAAMD,CAAW,EAErCC,EAAM,IAAI,QAAU,GACpBA,EAAM,KAAK,QAAQ,KAAO,yBAC1BA,EAAM,KAAK,QAAQ,WAAa,GAGhC,IAAMC,EAAM,CACV,MAAO,SAAS,eAAe,OAAO,EACtC,OAAQ,SAAS,eAAe,QAAQ,EACxC,IAAK,SAAS,eAAe,KAAK,EAClC,IAAK,SAAS,eAAe,QAAQ,EACrC,KAAM,SAAS,eAAe,aAAa,CAC7C,EACMC,EAAY,CAAE,OAAQ,EAAG,KAAM,EAAG,QAAS,EAAG,MAAO,CAAE,EACvDC,EAAM,CAAE,UAAW,EAAG,QAAS,EAAG,OAAQ,EAAG,UAAW,CAAE,EAE1DC,EAAM,IAAIC,IAAQ,CACtBJ,EAAI,IAAI,WAAaI,EAAI,KAAK,GAAG,EAAI;AAAA,EACrC,QAAQ,IAAI,GAAGA,CAAG,CACpB,EACMC,EAAUD,GAAQJ,EAAI,IAAI,UAAYI,EACtCE,EAAQF,GAAQJ,EAAI,KAAK,UAAY,WAAaD,EAAM,GAAG,OAAO,EAAE,WAAW,SAAS,EAAI,mBAAqB,KAAK,UAAUK,CAAG,EAAE,QAAQ,SAAU,EAAE,EAAE,QAAQ,KAAM,KAAK,EAEpL,eAAeG,GAAgB,CAC7B,GAAI,CAACP,EAAI,MAAM,OAAQ,CACjBC,EAAU,QAAU,IAAGA,EAAU,MAAQF,EAAM,IAAI,GAEvD,MAAMA,EAAM,OAAOC,EAAI,KAAK,EAC5B,IAAMQ,EAAUT,EAAM,GAAG,OAAO,EAAE,WAC9BS,EAAUP,EAAU,UAAY,GAAGE,EAAI,qBAAsBK,EAAUP,EAAU,OAAO,EAC5FA,EAAU,QAAUO,EACpBN,EAAI,UAAY,KAAK,MAAM,IAAO,KAAQH,EAAM,IAAI,EAAIE,EAAU,OAAO,EAAI,IAC7EC,EAAI,SACJA,EAAI,UAAY,KAAK,MAAM,KAAQH,EAAM,IAAI,EAAIE,EAAU,OAASC,EAAI,MAAM,EAAI,IAC9EA,EAAI,OAAS,MAAQ,GAAK,CAACF,EAAI,MAAM,QAAQG,EAAI,cAAe,CAAE,GAAGD,EAAK,QAASD,EAAU,OAAQ,CAAC,CAC5G,CACAA,EAAU,OAASF,EAAM,IAAI,EAC7B,sBAAsBQ,CAAa,CACrC,CAEA,eAAeE,GAAW,CAhE1B,IAAAC,EAAAC,EAAAC,EAiEE,GAAI,CAACZ,EAAI,MAAM,OAAQ,CACrB,IAAMa,EAAed,EAAM,KAAKA,EAAM,MAAM,EACxCA,EAAM,OAAO,OAAO,KAAMA,EAAM,KAAK,OAAOc,EAAa,OAA6Bb,EAAI,MAAM,EAC/FD,EAAM,KAAK,OAAOC,EAAI,MAAOA,EAAI,MAAM,EAE5C,IAAMc,EAA8B,CAAE,WAAY,kCAAiCF,GAAAD,GAAAD,EAAAX,EAAM,SAAN,YAAAW,EAAc,OAAd,YAAAC,EAAqB,KAArB,YAAAC,EAAyB,UAAU,kBAAmB,EACzI,MAAMb,EAAM,KAAK,IAAIC,EAAI,OAAQa,EAAcC,CAAG,EAClDR,EAAKO,EAAa,WAAW,CAC/B,CACA,IAAME,EAAMhB,EAAM,IAAI,EACtBG,EAAI,QAAU,KAAK,MAAM,IAAO,KAAQa,EAAMd,EAAU,KAAK,EAAI,IACjEA,EAAU,KAAOc,EACjBV,EAAOL,EAAI,MAAM,OAAS,SAAW,QAAQE,EAAI,UAAU,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,cAAcA,EAAI,QAAQ,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,QAAQ,EACjJ,WAAWO,EAAU,EAAE,CACzB,CAEA,eAAeO,GAAS,CACtB,MAAMjB,EAAM,OAAO,MAAM,CAAE,QAASC,EAAI,MAAO,KAAM,EAAK,CAAC,EAC3DA,EAAI,OAAO,MAAQD,EAAM,OAAO,MAChCC,EAAI,OAAO,OAASD,EAAM,OAAO,OACjCC,EAAI,OAAO,QAAU,SAAY,CAC3BD,EAAM,OAAO,OAAQ,MAAMA,EAAM,OAAO,KAAK,EAC5CA,EAAM,OAAO,MAAM,CAC1B,CACF,CAEA,eAAekB,GAAO,CACpBd,EAAI,iBAAkBJ,EAAM,QAAS,kBAAmBA,EAAM,GAAG,QAAQ,YAAY,EACrFI,EAAI,YAAaJ,EAAM,IAAI,SAAU,WAAYA,EAAM,IAAI,KAAK,EAChEM,EAAO,YAAY,EACnB,MAAMN,EAAM,KAAK,EACjBI,EAAI,WAAYJ,EAAM,GAAG,WAAW,EAAG,eAAgBA,EAAM,IAAI,QAAQ,EACzEI,EAAI,gBAAiBJ,EAAM,cAAc,CAAC,EAC1CI,EAAI,iBAAkB,OAAO,OAAOJ,EAAM,MAAM,EAAE,OAAQmB,GAAUA,IAAU,IAAI,EAAE,MAAM,EAC1Ff,EAAI,cAAeJ,EAAM,GAAG,EAC5BM,EAAO,iBAAiB,EACxB,MAAMN,EAAM,OAAO,EACnB,MAAMiB,EAAO,EACb,MAAMT,EAAc,EACpB,MAAME,EAAS,CACjB,CAEA,OAAO,OAASQ", - "names": ["H", "humanConfig", "human", "dom", "timestamp", "fps", "log", "msg", "status", "perf", "detectionLoop", "tensors", "drawLoop", "_a", "_b", "_c", "interpolated", "opt", "now", "webCam", "main", "model"] + "sourcesContent": ["/**\n * Human demo for browsers\n * @default Human Library\n * @summary \n * @author \n * @copyright \n * @license MIT\n */\n\nimport * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human\n\nconst width = 1920; // used by webcam config as well as human maximum resultion // can be anything, but resolutions higher than 4k will disable internal optimizations\n\nconst humanConfig: Partial = { // user configuration for human, used to fine-tune behavior\n // backend: 'webgpu',\n modelBasePath: '../../models',\n filter: { enabled: true, equalization: false, flip: false, width },\n face: { enabled: true, detector: { rotation: true }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } },\n body: { enabled: true },\n hand: { enabled: true },\n object: { enabled: false },\n segmentation: { enabled: false },\n gesture: { enabled: true },\n};\n\nconst human = new H.Human(humanConfig); // create instance of human with overrides from user configuration\n\nhuman.env.perfadd = false; // is performance data showing instant or total values\nhuman.draw.options.font = 'small-caps 18px \"Lato\"'; // set font used to draw labels when using draw methods\nhuman.draw.options.lineHeight = 20;\n// human.draw.options.fillPolygons = true;\n\nconst dom = { // grab instances of dom objects so we dont have to look them up later\n video: document.getElementById('video') as HTMLVideoElement,\n canvas: document.getElementById('canvas') as HTMLCanvasElement,\n log: document.getElementById('log') as HTMLPreElement,\n fps: document.getElementById('status') as HTMLPreElement,\n perf: document.getElementById('performance') as HTMLDivElement,\n};\nconst timestamp = { detect: 0, draw: 0, tensors: 0, start: 0 }; // holds information used to calculate performance and possible memory leaks\nconst fps = { detectFPS: 0, drawFPS: 0, frames: 0, averageMs: 0 }; // holds calculated fps information for both detect and screen refresh\n\nconst log = (...msg) => { // helper method to output messages\n dom.log.innerText += msg.join(' ') + '\\n';\n console.log(...msg); // eslint-disable-line no-console\n};\nconst status = (msg) => dom.fps.innerText = msg; // print status element\nconst perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors.toString() + ' | performance: ' + JSON.stringify(msg).replace(/\"|{|}/g, '').replace(/,/g, ' | '); // print performance element\n\nasync function detectionLoop() { // main detection loop\n if (!dom.video.paused) {\n if (timestamp.start === 0) timestamp.start = human.now();\n // log('profiling data:', await human.profile(dom.video));\n await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result\n const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks\n if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak\n timestamp.tensors = tensors;\n fps.detectFPS = Math.round(1000 * 1000 / (human.now() - timestamp.detect)) / 1000;\n fps.frames++;\n fps.averageMs = Math.round(1000 * (human.now() - timestamp.start) / fps.frames) / 1000;\n if (fps.frames % 100 === 0 && !dom.video.paused) log('performance', { ...fps, tensors: timestamp.tensors });\n }\n timestamp.detect = human.now();\n requestAnimationFrame(detectionLoop); // start new frame immediately\n}\n\nasync function drawLoop() { // main screen refresh loop\n if (!dom.video.paused) {\n const interpolated = human.next(human.result); // smoothen result using last-known results\n const processed = await human.image(dom.video); // get current video frame, but enhanced with human.filters\n human.draw.canvas(processed.canvas as HTMLCanvasElement, dom.canvas);\n\n const opt: Partial = { bodyLabels: `person confidence [score] and ${human.result?.body?.[0]?.keypoints.length} keypoints` };\n await human.draw.all(dom.canvas, interpolated, opt); // draw labels, boxes, lines, etc.\n perf(interpolated.performance); // write performance data\n }\n const now = human.now();\n fps.drawFPS = Math.round(1000 * 1000 / (now - timestamp.draw)) / 1000;\n timestamp.draw = now;\n status(dom.video.paused ? 'paused' : `fps: ${fps.detectFPS.toFixed(1).padStart(5, ' ')} detect | ${fps.drawFPS.toFixed(1).padStart(5, ' ')} draw`); // write status\n setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps\n}\n\nasync function webCam() {\n await human.webcam.start({ element: dom.video, crop: true, width }); // use human webcam helper methods and associate webcam stream with a dom element\n dom.canvas.width = human.webcam.width;\n dom.canvas.height = human.webcam.height;\n dom.canvas.onclick = async () => { // pause when clicked on screen and resume on next click\n if (human.webcam.paused) await human.webcam.play();\n else human.webcam.pause();\n };\n}\n\nasync function main() { // main entry point\n log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);\n log('platform:', human.env.platform, '| agent:', human.env.agent);\n status('loading...');\n await human.load(); // preload all models\n log('backend:', human.tf.getBackend(), '| available:', human.env.backends);\n log('models stats:', human.getModelStats());\n log('models loaded:', Object.values(human.models).filter((model) => model !== null).length);\n log('environment', human.env);\n status('initializing...');\n await human.warmup(); // warmup function to initialize backend for future faster detection\n await webCam(); // start webcam\n await detectionLoop(); // start detection loop\n await drawLoop(); // start draw loop\n}\n\nwindow.onload = main;\n"], + "mappings": ";;;;;;AASA,UAAYA,MAAO,0BAEnB,IAAMC,EAAQ,KAERC,EAAiC,CAErC,cAAe,eACf,OAAQ,CAAE,QAAS,GAAM,aAAc,GAAO,KAAM,GAAO,MAAAD,CAAM,EACjE,KAAM,CAAE,QAAS,GAAM,SAAU,CAAE,SAAU,EAAK,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,UAAW,CAAE,QAAS,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,YAAa,CAAE,QAAS,EAAK,EAAG,QAAS,CAAE,QAAS,EAAK,EAAG,UAAW,CAAE,QAAS,EAAK,EAAG,SAAU,CAAE,QAAS,EAAK,CAAE,EAC5P,KAAM,CAAE,QAAS,EAAK,EACtB,KAAM,CAAE,QAAS,EAAK,EACtB,OAAQ,CAAE,QAAS,EAAM,EACzB,aAAc,CAAE,QAAS,EAAM,EAC/B,QAAS,CAAE,QAAS,EAAK,CAC3B,EAEME,EAAQ,IAAM,QAAMD,CAAW,EAErCC,EAAM,IAAI,QAAU,GACpBA,EAAM,KAAK,QAAQ,KAAO,yBAC1BA,EAAM,KAAK,QAAQ,WAAa,GAGhC,IAAMC,EAAM,CACV,MAAO,SAAS,eAAe,OAAO,EACtC,OAAQ,SAAS,eAAe,QAAQ,EACxC,IAAK,SAAS,eAAe,KAAK,EAClC,IAAK,SAAS,eAAe,QAAQ,EACrC,KAAM,SAAS,eAAe,aAAa,CAC7C,EACMC,EAAY,CAAE,OAAQ,EAAG,KAAM,EAAG,QAAS,EAAG,MAAO,CAAE,EACvDC,EAAM,CAAE,UAAW,EAAG,QAAS,EAAG,OAAQ,EAAG,UAAW,CAAE,EAE1DC,EAAM,IAAIC,IAAQ,CACtBJ,EAAI,IAAI,WAAaI,EAAI,KAAK,GAAG,EAAI;AAAA,EACrC,QAAQ,IAAI,GAAGA,CAAG,CACpB,EACMC,EAAUD,GAAQJ,EAAI,IAAI,UAAYI,EACtCE,EAAQF,GAAQJ,EAAI,KAAK,UAAY,WAAaD,EAAM,GAAG,OAAO,EAAE,WAAW,SAAS,EAAI,mBAAqB,KAAK,UAAUK,CAAG,EAAE,QAAQ,SAAU,EAAE,EAAE,QAAQ,KAAM,KAAK,EAEpL,eAAeG,GAAgB,CAC7B,GAAI,CAACP,EAAI,MAAM,OAAQ,CACjBC,EAAU,QAAU,IAAGA,EAAU,MAAQF,EAAM,IAAI,GAEvD,MAAMA,EAAM,OAAOC,EAAI,KAAK,EAC5B,IAAMQ,EAAUT,EAAM,GAAG,OAAO,EAAE,WAC9BS,EAAUP,EAAU,UAAY,GAAGE,EAAI,qBAAsBK,EAAUP,EAAU,OAAO,EAC5FA,EAAU,QAAUO,EACpBN,EAAI,UAAY,KAAK,MAAM,IAAO,KAAQH,EAAM,IAAI,EAAIE,EAAU,OAAO,EAAI,IAC7EC,EAAI,SACJA,EAAI,UAAY,KAAK,MAAM,KAAQH,EAAM,IAAI,EAAIE,EAAU,OAASC,EAAI,MAAM,EAAI,IAC9EA,EAAI,OAAS,MAAQ,GAAK,CAACF,EAAI,MAAM,QAAQG,EAAI,cAAe,CAAE,GAAGD,EAAK,QAASD,EAAU,OAAQ,CAAC,CAC5G,CACAA,EAAU,OAASF,EAAM,IAAI,EAC7B,sBAAsBQ,CAAa,CACrC,CAEA,eAAeE,GAAW,CAlE1B,IAAAC,EAAAC,EAAAC,EAmEE,GAAI,CAACZ,EAAI,MAAM,OAAQ,CACrB,IAAMa,EAAed,EAAM,KAAKA,EAAM,MAAM,EACtCe,EAAY,MAAMf,EAAM,MAAMC,EAAI,KAAK,EAC7CD,EAAM,KAAK,OAAOe,EAAU,OAA6Bd,EAAI,MAAM,EAEnE,IAAMe,EAA8B,CAAE,WAAY,kCAAiCH,GAAAD,GAAAD,EAAAX,EAAM,SAAN,YAAAW,EAAc,OAAd,YAAAC,EAAqB,KAArB,YAAAC,EAAyB,UAAU,kBAAmB,EACzI,MAAMb,EAAM,KAAK,IAAIC,EAAI,OAAQa,EAAcE,CAAG,EAClDT,EAAKO,EAAa,WAAW,CAC/B,CACA,IAAMG,EAAMjB,EAAM,IAAI,EACtBG,EAAI,QAAU,KAAK,MAAM,IAAO,KAAQc,EAAMf,EAAU,KAAK,EAAI,IACjEA,EAAU,KAAOe,EACjBX,EAAOL,EAAI,MAAM,OAAS,SAAW,QAAQE,EAAI,UAAU,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,cAAcA,EAAI,QAAQ,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,QAAQ,EACjJ,WAAWO,EAAU,EAAE,CACzB,CAEA,eAAeQ,GAAS,CACtB,MAAMlB,EAAM,OAAO,MAAM,CAAE,QAASC,EAAI,MAAO,KAAM,GAAM,MAAAH,CAAM,CAAC,EAClEG,EAAI,OAAO,MAAQD,EAAM,OAAO,MAChCC,EAAI,OAAO,OAASD,EAAM,OAAO,OACjCC,EAAI,OAAO,QAAU,SAAY,CAC3BD,EAAM,OAAO,OAAQ,MAAMA,EAAM,OAAO,KAAK,EAC5CA,EAAM,OAAO,MAAM,CAC1B,CACF,CAEA,eAAemB,GAAO,CACpBf,EAAI,iBAAkBJ,EAAM,QAAS,kBAAmBA,EAAM,GAAG,QAAQ,YAAY,EACrFI,EAAI,YAAaJ,EAAM,IAAI,SAAU,WAAYA,EAAM,IAAI,KAAK,EAChEM,EAAO,YAAY,EACnB,MAAMN,EAAM,KAAK,EACjBI,EAAI,WAAYJ,EAAM,GAAG,WAAW,EAAG,eAAgBA,EAAM,IAAI,QAAQ,EACzEI,EAAI,gBAAiBJ,EAAM,cAAc,CAAC,EAC1CI,EAAI,iBAAkB,OAAO,OAAOJ,EAAM,MAAM,EAAE,OAAQoB,GAAUA,IAAU,IAAI,EAAE,MAAM,EAC1FhB,EAAI,cAAeJ,EAAM,GAAG,EAC5BM,EAAO,iBAAiB,EACxB,MAAMN,EAAM,OAAO,EACnB,MAAMkB,EAAO,EACb,MAAMV,EAAc,EACpB,MAAME,EAAS,CACjB,CAEA,OAAO,OAASS", + "names": ["H", "width", "humanConfig", "human", "dom", "timestamp", "fps", "log", "msg", "status", "perf", "detectionLoop", "tensors", "drawLoop", "_a", "_b", "_c", "interpolated", "processed", "opt", "now", "webCam", "main", "model"] } diff --git a/demo/typescript/index.ts b/demo/typescript/index.ts index 0699b9cf..a127491d 100644 --- a/demo/typescript/index.ts +++ b/demo/typescript/index.ts @@ -9,11 +9,13 @@ import * as H from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human +const width = 1920; // used by webcam config as well as human maximum resultion // can be anything, but resolutions higher than 4k will disable internal optimizations + const humanConfig: Partial = { // user configuration for human, used to fine-tune behavior - // backend: 'wasm', + // backend: 'webgpu', modelBasePath: '../../models', - filter: { enabled: true, equalization: false, flip: false }, - face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } }, + filter: { enabled: true, equalization: false, flip: false, width }, + face: { enabled: true, detector: { rotation: true }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true }, antispoof: { enabled: true }, liveness: { enabled: true } }, body: { enabled: true }, hand: { enabled: true }, object: { enabled: false }, @@ -65,8 +67,8 @@ async function detectionLoop() { // main detection loop async function drawLoop() { // main screen refresh loop if (!dom.video.paused) { const interpolated = human.next(human.result); // smoothen result using last-known results - if (human.config.filter.flip) human.draw.canvas(interpolated.canvas as HTMLCanvasElement, dom.canvas); // draw processed image to screen canvas - else human.draw.canvas(dom.video, dom.canvas); // draw original video to screen canvas // better than using procesed image as this loop happens faster than processing loop + const processed = await human.image(dom.video); // get current video frame, but enhanced with human.filters + human.draw.canvas(processed.canvas as HTMLCanvasElement, dom.canvas); const opt: Partial = { bodyLabels: `person confidence [score] and ${human.result?.body?.[0]?.keypoints.length} keypoints` }; await human.draw.all(dom.canvas, interpolated, opt); // draw labels, boxes, lines, etc. @@ -80,7 +82,7 @@ async function drawLoop() { // main screen refresh loop } async function webCam() { - await human.webcam.start({ element: dom.video, crop: true }); // use human webcam helper methods and associate webcam stream with a dom element + await human.webcam.start({ element: dom.video, crop: true, width }); // use human webcam helper methods and associate webcam stream with a dom element dom.canvas.width = human.webcam.width; dom.canvas.height = human.webcam.height; dom.canvas.onclick = async () => { // pause when clicked on screen and resume on next click diff --git a/package.json b/package.json index b3633a18..41087844 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "devDependencies": { "@html-eslint/eslint-plugin": "^0.15.0", "@html-eslint/parser": "^0.15.0", - "@microsoft/api-extractor": "^7.33.5", + "@microsoft/api-extractor": "^7.33.6", "@tensorflow/tfjs-backend-cpu": "^4.0.0", "@tensorflow/tfjs-backend-wasm": "^4.0.0", "@tensorflow/tfjs-backend-webgl": "^4.0.0", @@ -85,14 +85,14 @@ "@tensorflow/tfjs-node-gpu": "^4.0.0", "@types/node": "^18.11.9", "@types/offscreencanvas": "^2019.7.0", - "@typescript-eslint/eslint-plugin": "^5.42.0", - "@typescript-eslint/parser": "^5.42.0", + "@typescript-eslint/eslint-plugin": "^5.42.1", + "@typescript-eslint/parser": "^5.42.1", "@vladmandic/build": "^0.7.14", "@vladmandic/pilogger": "^0.4.6", "@vladmandic/tfjs": "github:vladmandic/tfjs", "canvas": "^2.10.2", "esbuild": "^0.15.13", - "eslint": "8.26.0", + "eslint": "8.27.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-html": "^7.1.0", "eslint-plugin-import": "^2.26.0", diff --git a/src/config.ts b/src/config.ts index 08ff32c8..3dd4909d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -186,6 +186,8 @@ export interface FilterConfig { return: boolean, /** flip input as mirror image */ flip: boolean, + /** apply auto-brighness */ + autoBrightness: boolean, /** range: -1 (darken) to 1 (lighten) */ brightness: number, /** range: -1 (reduce contrast) to 1 (increase contrast) */ @@ -350,6 +352,7 @@ const config: Config = { height: 0, flip: false, return: true, + autoBrightness: true, brightness: 0, contrast: 0, sharpness: 0, diff --git a/src/human.ts b/src/human.ts index 46e03e9f..b036db38 100644 --- a/src/human.ts +++ b/src/human.ts @@ -139,7 +139,6 @@ export class Human { const tfVersion = (tf.version.tfjs || tf.version_core).replace(/-(.*)/, ''); defaults.wasmPath = `https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@${tfVersion}/dist/`; defaults.modelBasePath = env.browser ? '../models/' : 'file://models/'; - defaults.backend = env.browser ? 'webgl' : 'tensorflow'; this.version = app.version; // expose version property on instance of class Object.defineProperty(this, 'version', { value: app.version }); // expose version property directly on class itself this.config = JSON.parse(JSON.stringify(defaults)); @@ -252,7 +251,7 @@ export class Human { * @param getTensor - should image processing also return tensor or just canvas * Returns object with `tensor` and `canvas` */ - image(input: Input, getTensor: boolean = true) { + image(input: Input, getTensor: boolean = false) { return image.process(input, this.config, getTensor); } @@ -455,6 +454,7 @@ export class Human { timeStamp = now(); this.config.skipAllowed = await image.skip(this.config, img.tensor); + this.config.filter.autoBrightness = (this.config.filter.autoBrightness || false) && this.config.skipAllowed; // disable autoBrightness on scene change if (!this.performance.totalFrames) this.performance.totalFrames = 0; if (!this.performance.cachedFrames) this.performance.cachedFrames = 0; (this.performance.totalFrames)++; diff --git a/src/image/enhance.ts b/src/image/enhance.ts index 97846907..e807633a 100644 --- a/src/image/enhance.ts +++ b/src/image/enhance.ts @@ -6,19 +6,23 @@ import * as tf from 'dist/tfjs.esm.js'; import type { Tensor } from '../exports'; export async function histogramEqualization(inputImage: Tensor): Promise { - // const maxValue = 254; // using 255 results in values slightly larger than 1 due to math rounding errors const squeeze = inputImage.shape.length === 4 ? tf.squeeze(inputImage) : inputImage; - const channels = tf.split(squeeze, 3, 2); - const min: Tensor[] = [tf.min(channels[0]), tf.min(channels[1]), tf.min(channels[2])]; - const max: Tensor[] = [tf.max(channels[0]), tf.max(channels[1]), tf.max(channels[2])]; - const absMax = await Promise.all(max.map((channel) => channel.data())); - const maxValue = 0.99 * Math.max(absMax[0][0], absMax[1][0], absMax[2][0]); - const sub = [tf.sub(channels[0], min[0]), tf.sub(channels[1], min[1]), tf.sub(channels[2], min[2])]; - const range = [tf.sub(max[0], min[0]), tf.sub(max[1], min[1]), tf.sub(max[2], min[2])]; - const fact = [tf.div(maxValue, range[0]), tf.div(maxValue, range[1]), tf.div(maxValue, range[2])]; - const enh = [tf.mul(sub[0], fact[0]), tf.mul(sub[1], fact[1]), tf.mul(sub[2], fact[2])]; - const rgb = tf.stack([enh[0], enh[1], enh[2]], 2); - const reshape = tf.reshape(rgb, [1, squeeze.shape[0] || 0, squeeze.shape[1] || 0, 3]); - tf.dispose([...channels, ...min, ...max, ...sub, ...range, ...fact, ...enh, rgb, squeeze]); - return reshape; // output shape is [1, height, width, 3] + const rgb = tf.split(squeeze, 3, 2); + const min: Tensor[] = [tf.min(rgb[0]), tf.min(rgb[1]), tf.min(rgb[2])]; // minimum pixel value per channel T[] + const max: Tensor[] = [tf.max(rgb[0]), tf.max(rgb[1]), tf.max(rgb[2])]; // maximum pixel value per channel T[] + // const absMin = await Promise.all(min.map((channel) => channel.data())); // minimum pixel value per channel A[] + // const minValue = Math.min(absMax[0][0], absMin[1][0], absMin[2][0]); + const absMax = await Promise.all(max.map((channel) => channel.data())); // maximum pixel value per channel A[] + const maxValue = Math.max(absMax[0][0], absMax[1][0], absMax[2][0]); + const maxRange = maxValue > 1 ? 255 : 1; + const factor = maxRange / maxValue; + const sub = [tf.sub(rgb[0], min[0]), tf.sub(rgb[1], min[1]), tf.sub(rgb[2], min[2])]; // channels offset by min values + const range = [tf.sub(max[0], min[0]), tf.sub(max[1], min[1]), tf.sub(max[2], min[2])]; // channel ranges + // const fact = [tf.div(maxRange, absMax[0]), tf.div(maxRange, absMax[1]), tf.div(maxRange, absMax[1])]; // factors between + const enh = [tf.mul(sub[0], factor), tf.mul(sub[1], factor), tf.mul(sub[2], factor)]; + const stack = tf.stack([enh[0], enh[1], enh[2]], 2); + const reshape = tf.reshape(stack, [1, squeeze.shape[0] || 0, squeeze.shape[1] || 0, 3]); + const final = tf.squeeze(reshape); + tf.dispose([...rgb, ...min, ...max, ...sub, ...range, ...enh, rgb, squeeze, reshape]); + return final; // output shape is [height, width, 3] } diff --git a/src/image/image.ts b/src/image/image.ts index 2236adae..c5a4f85e 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -188,7 +188,7 @@ export async function process(input: Input, config: Config, getTensor: boolean = if (config.filter.technicolor) fx.add('technicolor'); if (config.filter.polaroid) fx.add('polaroid'); if (config.filter.pixelate !== 0) fx.add('pixelate', config.filter.pixelate); - if (fx.get() > 0) outCanvas = fx.apply(inCanvas); + if (fx.get()?.length > 1) outCanvas = fx.apply(inCanvas); else outCanvas = fx.draw(inCanvas); } } else { @@ -238,6 +238,14 @@ export async function process(input: Input, config: Config, getTensor: boolean = if (!pixels) throw new Error('input error: cannot create tensor'); const casted: Tensor = tf.cast(pixels, 'float32'); const tensor: Tensor = config.filter.equalization ? await enhance.histogramEqualization(casted) : tf.expandDims(casted, 0); + + if (config.filter.autoBrightness) { + const max = tf.max(tensor); + const maxVal = await max.data(); + config.filter.brightness = maxVal[0] > 1 ? (1 - maxVal[0] / 255) : (1 - maxVal[0]); + tf.dispose(max); + } + tf.dispose([pixels, casted]); return { tensor: tensor as Tensor4D, canvas: (config.filter.return ? outCanvas : null) }; } @@ -274,7 +282,7 @@ const checksum = async (input: Tensor): Promise => { // use tf sum or js export async function skip(config: Partial, input: Tensor) { let skipFrame = false; - if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 2048 || input.shape[2] > 2048) return skipFrame; // cache disabled or input is invalid or too large for cache analysis + if (config.cacheSensitivity === 0 || !input.shape || input.shape.length !== 4 || input.shape[1] > 3840 || input.shape[2] > 2160) return skipFrame; // cache disabled or input is invalid or too large for cache analysis /* const checkSum = await checksum(input); diff --git a/src/object/centernet.ts b/src/object/centernet.ts index de98b798..8c99eb39 100644 --- a/src/object/centernet.ts +++ b/src/object/centernet.ts @@ -1,7 +1,7 @@ /** * CenterNet object detection model implementation * - * Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet) + * Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet) */ import * as tf from 'dist/tfjs.esm.js'; diff --git a/src/object/nanodet.ts b/src/object/nanodet.ts index d8863edf..f1006c9c 100644 --- a/src/object/nanodet.ts +++ b/src/object/nanodet.ts @@ -1,7 +1,7 @@ /** * NanoDet object detection model implementation * - * Based on: [**MB3-CenterNet**](https://github.com/610265158/mobilenetv3_centernet) + * Based on: [**NanoDet**](https://github.com/RangiLyu/nanodet) */ import * as tf from 'dist/tfjs.esm.js'; diff --git a/src/tfjs/backend.ts b/src/tfjs/backend.ts index cdbc4356..603e44b9 100644 --- a/src/tfjs/backend.ts +++ b/src/tfjs/backend.ts @@ -8,6 +8,15 @@ import * as humangl from './humangl'; import * as constants from './constants'; import type { TensorInfo } from './types'; +export async function getBestBackend(): Promise { + await env.updateBackend(); // update env on backend init + if (!env.browser) return 'tensorflow'; + if (env.webgpu.supported && env.webgpu.backend) return 'webgpu'; + if (env.webgl.supported && env.webgl.backend) return 'webgl'; + if (env.wasm.supported && env.wasm.backend) return 'wasm'; + return 'cpu'; +} + function registerCustomOps(config: Config) { const newKernels: string[] = []; if (!env.kernels.includes('mod')) { @@ -73,6 +82,7 @@ let defaultFlags: Record = {}; export async function check(instance: Human, force = false) { instance.state = 'backend'; + if (instance.config.backend?.length === 0) instance.config.backend = await getBestBackend(); if (force || env.initial || (instance.config.backend && (instance.config.backend.length > 0) && (tf.getBackend() !== instance.config.backend))) { const timeStamp = now(); diff --git a/src/util/env.ts b/src/util/env.ts index e8ac6291..996b401f 100644 --- a/src/util/env.ts +++ b/src/util/env.ts @@ -53,17 +53,21 @@ export class Env { backend: undefined | boolean, version: undefined | string, renderer: undefined | string, + shader: undefined | string, + vendor: undefined | string, } = { supported: undefined, backend: undefined, version: undefined, renderer: undefined, + shader: undefined, + vendor: undefined, }; /** WebGPU detected capabilities */ webgpu: { supported: undefined | boolean, backend: undefined | boolean, - adapter: undefined | string, + adapter: undefined | GPUAdapterInfo, } = { supported: undefined, backend: undefined, @@ -123,35 +127,36 @@ export class Env { async updateBackend() { // analyze backends this.backends = Object.keys(tf.engine().registryFactory); - this.tensorflow = { - version: (tf.backend()['binding'] ? tf.backend()['binding'].TF_Version : undefined), - gpu: (tf.backend()['binding'] ? tf.backend()['binding'].isUsingGpuDevice() : undefined), - }; + try { + this.tensorflow = { + version: (tf.backend()['binding'] ? tf.backend()['binding'].TF_Version : undefined), + gpu: (tf.backend()['binding'] ? tf.backend()['binding'].isUsingGpuDevice() : undefined), + }; + } catch { /**/ } this.wasm.supported = typeof WebAssembly !== 'undefined'; this.wasm.backend = this.backends.includes('wasm'); - if (this.wasm.supported && this.wasm.backend && tf.getBackend() === 'wasm') { - this.wasm.simd = tf.env().get('WASM_HAS_SIMD_SUPPORT') as boolean; - this.wasm.multithread = tf.env().get('WASM_HAS_MULTITHREAD_SUPPORT') as boolean; + if (this.wasm.supported && this.wasm.backend) { + this.wasm.simd = await tf.env().getAsync('WASM_HAS_SIMD_SUPPORT') as boolean; + this.wasm.multithread = await tf.env().getAsync('WASM_HAS_MULTITHREAD_SUPPORT') as boolean; } const c = image.canvas(100, 100); const ctx = c ? c.getContext('webgl2') : undefined; // causes too many gl contexts // const ctx = typeof tf.backend().getGPGPUContext !== undefined ? tf.backend().getGPGPUContext : null; this.webgl.supported = typeof ctx !== 'undefined'; this.webgl.backend = this.backends.includes('webgl'); - if (this.webgl.supported && this.webgl.backend && (tf.getBackend() === 'webgl' || tf.getBackend() === 'humangl')) { - const backend = tf.backend(); - const gl = typeof backend['gpgpu'] !== 'undefined' ? backend['getGPGPUContext']().gl : null; - if (gl) { - this.webgl.version = gl.getParameter(gl.VERSION); - this.webgl.renderer = gl.getParameter(gl.RENDERER); - } + if (this.webgl.supported && this.webgl.backend) { + const gl = ctx as WebGL2RenderingContext; + this.webgl.version = gl.getParameter(gl.VERSION); + this.webgl.vendor = gl.getParameter(gl.VENDOR); + this.webgl.renderer = gl.getParameter(gl.RENDERER); + this.webgl.shader = gl.getParameter(gl.SHADING_LANGUAGE_VERSION); } this.webgpu.supported = this.browser && typeof navigator.gpu !== 'undefined'; this.webgpu.backend = this.backends.includes('webgpu'); try { if (this.webgpu.supported) { const adapter = await navigator.gpu.requestAdapter(); - this.webgpu.adapter = adapter ? adapter.name : undefined; + this.webgpu.adapter = await adapter?.requestAdapterInfo(); } } catch { this.webgpu.supported = false; diff --git a/test/build.log b/test/build.log index 1bcf00e3..e512420a 100644 --- a/test/build.log +++ b/test/build.log @@ -1,40 +1,40 @@ -2022-11-04 13:19:02 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"} -2022-11-04 13:19:02 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"} -2022-11-04 13:19:02 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} -2022-11-04 13:19:02 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.13","typescript":"4.8.4","typedoc":"0.23.20","eslint":"8.26.0"} -2022-11-04 13:19:02 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} -2022-11-04 13:19:02 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":79,"inputBytes":672790,"outputBytes":316303} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":79,"inputBytes":672794,"outputBytes":316307} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":79,"inputBytes":673742,"outputBytes":316418} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":79,"inputBytes":672536,"outputBytes":314908} -2022-11-04 13:19:02 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144900} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":79,"inputBytes":1816766,"outputBytes":1456466} -2022-11-04 13:19:02 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":79,"inputBytes":1816766,"outputBytes":1913830} -2022-11-04 13:19:06 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} -2022-11-04 13:19:08 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true} -2022-11-04 13:19:08 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5936,"outputBytes":2867} -2022-11-04 13:19:08 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17134,"outputBytes":9181} -2022-11-04 13:19:16 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":116,"errors":0,"warnings":0} -2022-11-04 13:19:16 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} -2022-11-04 13:19:16 STATE: Copy: {"input":"src/tfjs","output":"dist/tfjs.esm.d.ts"} -2022-11-04 13:19:16 INFO:  Done... -2022-11-04 13:19:17 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195} -2022-11-04 13:19:17 STATE: Filter: {"input":"types/human.d.ts"} -2022-11-04 13:19:17 STATE: Link: {"input":"types/human.d.ts"} -2022-11-04 13:19:17 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} -2022-11-04 13:19:17 STATE: Models {"folder":"./models","models":12} -2022-11-04 13:19:17 STATE: Models {"folder":"../human-models/models","models":43} -2022-11-04 13:19:17 STATE: Models {"folder":"../blazepose/model/","models":4} -2022-11-04 13:19:17 STATE: Models {"folder":"../anti-spoofing/model","models":1} -2022-11-04 13:19:17 STATE: Models {"folder":"../efficientpose/models","models":3} -2022-11-04 13:19:17 STATE: Models {"folder":"../insightface/models","models":5} -2022-11-04 13:19:17 STATE: Models {"folder":"../movenet/models","models":3} -2022-11-04 13:19:17 STATE: Models {"folder":"../nanodet/models","models":4} -2022-11-04 13:19:18 STATE: Models: {"count":58,"totalSize":386543911} -2022-11-04 13:19:18 INFO:  Human Build complete... {"logFile":"test/build.log"} +2022-11-10 20:16:03 DATA:  Build {"name":"@vladmandic/human","version":"3.0.0"} +2022-11-10 20:16:03 INFO:  Application: {"name":"@vladmandic/human","version":"3.0.0"} +2022-11-10 20:16:03 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} +2022-11-10 20:16:03 INFO:  Toolchain: {"build":"0.7.14","esbuild":"0.15.13","typescript":"4.8.4","typedoc":"0.23.20","eslint":"8.27.0"} +2022-11-10 20:16:03 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} +2022-11-10 20:16:03 STATE: Clean: {"locations":["dist/*","types/*","typedoc/*"]} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1289,"outputBytes":361} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":569,"outputBytes":924} +2022-11-10 20:16:03 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":79,"inputBytes":674094,"outputBytes":316829} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":577,"outputBytes":928} +2022-11-10 20:16:03 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":79,"inputBytes":674098,"outputBytes":316833} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":665,"outputBytes":1876} +2022-11-10 20:16:03 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":79,"inputBytes":675046,"outputBytes":316944} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1375,"outputBytes":670} +2022-11-10 20:16:03 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":79,"inputBytes":673840,"outputBytes":315438} +2022-11-10 20:16:03 STATE: Compile: {"name":"tfjs/browser/esm/bundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":10,"inputBytes":1375,"outputBytes":1144900} +2022-11-10 20:16:03 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":79,"inputBytes":1818070,"outputBytes":1457034} +2022-11-10 20:16:04 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":79,"inputBytes":1818070,"outputBytes":1914674} +2022-11-10 20:16:08 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":15} +2022-11-10 20:16:10 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":76,"generated":true} +2022-11-10 20:16:10 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5981,"outputBytes":2862} +2022-11-10 20:16:10 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":17134,"outputBytes":9181} +2022-11-10 20:16:19 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":116,"errors":0,"warnings":0} +2022-11-10 20:16:19 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} +2022-11-10 20:16:19 STATE: Copy: {"input":"src/tfjs","output":"dist/tfjs.esm.d.ts"} +2022-11-10 20:16:19 INFO:  Done... +2022-11-10 20:16:20 STATE: API-Extractor: {"succeeeded":true,"errors":0,"warnings":195} +2022-11-10 20:16:20 STATE: Filter: {"input":"types/human.d.ts"} +2022-11-10 20:16:20 STATE: Link: {"input":"types/human.d.ts"} +2022-11-10 20:16:20 INFO:  Analyze models: {"folders":8,"result":"models/models.json"} +2022-11-10 20:16:20 STATE: Models {"folder":"./models","models":12} +2022-11-10 20:16:20 STATE: Models {"folder":"../human-models/models","models":43} +2022-11-10 20:16:20 STATE: Models {"folder":"../blazepose/model/","models":4} +2022-11-10 20:16:20 STATE: Models {"folder":"../anti-spoofing/model","models":1} +2022-11-10 20:16:20 STATE: Models {"folder":"../efficientpose/models","models":3} +2022-11-10 20:16:20 STATE: Models {"folder":"../insightface/models","models":5} +2022-11-10 20:16:20 STATE: Models {"folder":"../movenet/models","models":3} +2022-11-10 20:16:20 STATE: Models {"folder":"../nanodet/models","models":4} +2022-11-10 20:16:21 STATE: Models: {"count":58,"totalSize":386543911} +2022-11-10 20:16:21 INFO:  Human Build complete... {"logFile":"test/build.log"}