diff --git a/CHANGELOG.md b/CHANGELOG.md index df256aee..2b82938c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # @vladmandic/human - Version: **2.6.5** + Version: **2.7.0** Description: **Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition** Author: **Vladimir Mandic ** @@ -9,8 +9,9 @@ ## Changelog -### **HEAD -> main** 2022/04/05 mandic00@live.com +### **HEAD -> main** 2022/04/10 mandic00@live.com +- rebuild ### **2.6.5** 2022/04/01 mandic00@live.com diff --git a/demo/typescript/index.html b/demo/typescript/index.html index a74a14bc..47296bfb 100644 --- a/demo/typescript/index.html +++ b/demo/typescript/index.html @@ -25,6 +25,6 @@

     

-    
+
diff --git a/demo/typescript/index.js b/demo/typescript/index.js index 24d94078..d1025ce8 100644 --- a/demo/typescript/index.js +++ b/demo/typescript/index.js @@ -4,8 +4,8 @@ author: ' */ -import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!0},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+` -`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],u=o.getCapabilities?o.getCapabilities():"",v=o.getSettings?o.getSettings():"",g=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:v,constraints:g,capabilities:u}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y; +import{Human as p}from"../../dist/human.esm.js";var w={modelBasePath:"../../models",filter:{enabled:!0,equalization:!1},cacheSensitivity:0,face:{enabled:!0,detector:{rotation:!1},mesh:{enabled:!0},attention:{enabled:!1},iris:{enabled:!0},description:{enabled:!0},emotion:{enabled:!0}},body:{enabled:!0},hand:{enabled:!0},object:{enabled:!0},gesture:{enabled:!0}},t=new p(w);t.env.perfadd=!1;t.draw.options.font='small-caps 18px "Lato"';t.draw.options.lineHeight=20;var e={video:document.getElementById("video"),canvas:document.getElementById("canvas"),log:document.getElementById("log"),fps:document.getElementById("status"),perf:document.getElementById("performance")},i={detect:0,draw:0,tensors:0},d={detect:0,draw:0},s=(...a)=>{e.log.innerText+=a.join(" ")+` +`,console.log(...a)},r=a=>e.fps.innerText=a,b=a=>e.perf.innerText="tensors:"+t.tf.memory().numTensors+" | performance: "+JSON.stringify(a).replace(/"|{|}/g,"").replace(/,/g," | ");async function h(){r("starting webcam...");let a={audio:!1,video:{facingMode:"user",resizeMode:"none",width:{ideal:document.body.clientWidth}}},n=await navigator.mediaDevices.getUserMedia(a),m=new Promise(f=>{e.video.onloadeddata=()=>f(!0)});e.video.srcObject=n,e.video.play(),await m,e.canvas.width=e.video.videoWidth,e.canvas.height=e.video.videoHeight;let o=n.getVideoTracks()[0],v=o.getCapabilities?o.getCapabilities():"",u=o.getSettings?o.getSettings():"",g=o.getConstraints?o.getConstraints():"";s("video:",e.video.videoWidth,e.video.videoHeight,o.label,{stream:n,track:o,settings:u,constraints:g,capabilities:v}),e.canvas.onclick=()=>{e.video.paused?e.video.play():e.video.pause()}}async function c(){if(!e.video.paused){await t.detect(e.video);let n=t.tf.memory().numTensors;n-i.tensors!==0&&s("allocated tensors:",n-i.tensors),i.tensors=n}let a=t.now();d.detect=1e3/(a-i.detect),i.detect=a,requestAnimationFrame(c)}async function l(){if(!e.video.paused){let n=await t.next(t.result);await t.draw.canvas(e.video,e.canvas),await t.draw.all(e.canvas,n),b(n.performance)}let a=t.now();d.draw=1e3/(a-i.draw),i.draw=a,r(e.video.paused?"paused":`fps: ${d.detect.toFixed(1).padStart(5," ")} detect | ${d.draw.toFixed(1).padStart(5," ")} draw`),setTimeout(l,30)}async function y(){s("human version:",t.version,"| tfjs version:",t.tf.version["tfjs-core"]),s("platform:",t.env.platform,"| agent:",t.env.agent),r("loading..."),await t.load(),s("backend:",t.tf.getBackend(),"| available:",t.env.backends),s("loaded models:",Object.values(t.models).filter(a=>a!==null).length),r("initializing..."),await t.warmup(),await h(),await c(),await l()}window.onload=y; /** * Human demo for browsers * @default Human Library diff --git a/demo/typescript/index.js.map b/demo/typescript/index.js.map index 518c681f..3c244fca 100644 --- a/demo/typescript/index.js.map +++ b/demo/typescript/index.js.map @@ -1,7 +1,7 @@ { "version": 3, "sources": ["index.ts"], - "sourcesContent": ["/**\n * Human demo for browsers\n * @default Human Library\n * @summary \n * @author \n * @copyright \n * @license MIT\n */\n\nimport { Human, Config } from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human\n\nconst humanConfig: Partial = { // user configuration for human, used to fine-tune behavior\n // backend: 'webgpu' as const,\n // async: true,\n modelBasePath: '../../models',\n filter: { enabled: true, equalization: false },\n face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },\n body: { enabled: true },\n hand: { enabled: true },\n object: { enabled: true },\n gesture: { enabled: true },\n};\n\nconst human = new Human(humanConfig); // create instance of human with overrides from user configuration\n\nhuman.env['perfadd'] = false; // is performance data showing instant or total values\nhuman.draw.options.font = 'small-caps 18px \"Lato\"'; // set font used to draw labels when using draw methods\nhuman.draw.options.lineHeight = 20;\n\nconst dom = { // grab instances of dom objects so we dont have to look them up later\n video: document.getElementById('video') as HTMLVideoElement,\n canvas: document.getElementById('canvas') as HTMLCanvasElement,\n log: document.getElementById('log') as HTMLPreElement,\n fps: document.getElementById('status') as HTMLPreElement,\n perf: document.getElementById('performance') as HTMLDivElement,\n};\nconst timestamp = { detect: 0, draw: 0, tensors: 0 }; // holds information used to calculate performance and possible memory leaks\nconst fps = { detect: 0, draw: 0 }; // holds calculated fps information for both detect and screen refresh\n\nconst log = (...msg) => { // helper method to output messages\n dom.log.innerText += msg.join(' ') + '\\n';\n // eslint-disable-next-line no-console\n console.log(...msg);\n};\nconst status = (msg) => dom.fps.innerText = msg; // print status element\nconst perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors + ' | performance: ' + JSON.stringify(msg).replace(/\"|{|}/g, '').replace(/,/g, ' | '); // print performance element\n\nasync function webCam() { // initialize webcam\n status('starting webcam...');\n // @ts-ignore resizeMode is not yet defined in tslib\n const options: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };\n const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);\n const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });\n dom.video.srcObject = stream;\n dom.video.play();\n await ready;\n dom.canvas.width = dom.video.videoWidth;\n dom.canvas.height = dom.video.videoHeight;\n const track: MediaStreamTrack = stream.getVideoTracks()[0];\n const capabilities: MediaTrackCapabilities | string = track.getCapabilities ? track.getCapabilities() : '';\n const settings: MediaTrackSettings | string = track.getSettings ? track.getSettings() : '';\n const constraints: MediaTrackConstraints | string = track.getConstraints ? track.getConstraints() : '';\n log('video:', dom.video.videoWidth, dom.video.videoHeight, track.label, { stream, track, settings, constraints, capabilities });\n dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click\n if (dom.video.paused) dom.video.play();\n else dom.video.pause();\n };\n}\n\nasync function detectionLoop() { // main detection loop\n if (!dom.video.paused) {\n // console.log('profiling data:', await human.profile(dom.video));\n await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result\n const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks\n if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak\n timestamp.tensors = tensors;\n }\n const now = human.now();\n fps.detect = 1000 / (now - timestamp.detect);\n timestamp.detect = now;\n requestAnimationFrame(detectionLoop); // start new frame immediately\n}\n\nasync function drawLoop() { // main screen refresh loop\n if (!dom.video.paused) {\n const interpolated = await human.next(human.result); // smoothen result using last-known results\n await human.draw.canvas(dom.video, dom.canvas); // draw canvas to screen\n await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.\n perf(interpolated.performance); // write performance data\n }\n const now = human.now();\n fps.draw = 1000 / (now - timestamp.draw);\n timestamp.draw = now;\n status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect | ${fps.draw.toFixed(1).padStart(5, ' ')} draw`); // write status\n // requestAnimationFrame(drawLoop); // refresh at screen refresh rate\n setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps\n}\n\nasync function main() { // main entry point\n log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);\n log('platform:', human.env.platform, '| agent:', human.env.agent);\n status('loading...');\n await human.load(); // preload all models\n log('backend:', human.tf.getBackend(), '| available:', human.env.backends);\n log('loaded models:', Object.values(human.models).filter((model) => model !== null).length);\n status('initializing...');\n await human.warmup(); // warmup function to initialize backend for future faster detection\n await webCam(); // start webcam\n await detectionLoop(); // start detection loop\n await drawLoop(); // start draw loop\n}\n\nwindow.onload = main;\n"], - "mappings": ";;;;;;AASA,gDATA,AAWA,GAAM,GAA+B,CAGnC,cAAe,eACf,OAAQ,CAAE,QAAS,GAAM,aAAc,EAAM,EAC7C,KAAM,CAAE,QAAS,GAAM,SAAU,CAAE,SAAU,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,YAAa,CAAE,QAAS,EAAK,EAAG,QAAS,CAAE,QAAS,EAAK,CAAE,EACnK,KAAM,CAAE,QAAS,EAAK,EACtB,KAAM,CAAE,QAAS,EAAK,EACtB,OAAQ,CAAE,QAAS,EAAK,EACxB,QAAS,CAAE,QAAS,EAAK,CAC3B,EAEM,EAAQ,GAAI,GAAM,CAAW,EAEnC,EAAM,IAAI,QAAa,GACvB,EAAM,KAAK,QAAQ,KAAO,yBAC1B,EAAM,KAAK,QAAQ,WAAa,GAEhC,GAAM,GAAM,CACV,MAAO,SAAS,eAAe,OAAO,EACtC,OAAQ,SAAS,eAAe,QAAQ,EACxC,IAAK,SAAS,eAAe,KAAK,EAClC,IAAK,SAAS,eAAe,QAAQ,EACrC,KAAM,SAAS,eAAe,aAAa,CAC7C,EACM,EAAY,CAAE,OAAQ,EAAG,KAAM,EAAG,QAAS,CAAE,EAC7C,EAAM,CAAE,OAAQ,EAAG,KAAM,CAAE,EAE3B,EAAM,IAAI,IAAQ,CACtB,EAAI,IAAI,WAAa,EAAI,KAAK,GAAG,EAAI;AAAA,EAErC,QAAQ,IAAI,GAAG,CAAG,CACpB,EACM,EAAS,AAAC,GAAQ,EAAI,IAAI,UAAY,EACtC,EAAO,AAAC,GAAQ,EAAI,KAAK,UAAY,WAAa,EAAM,GAAG,OAAO,EAAE,WAAa,mBAAqB,KAAK,UAAU,CAAG,EAAE,QAAQ,SAAU,EAAE,EAAE,QAAQ,KAAM,KAAK,EAEzK,kBAAwB,CACtB,EAAO,oBAAoB,EAE3B,GAAM,GAAkC,CAAE,MAAO,GAAO,MAAO,CAAE,WAAY,OAAQ,WAAY,OAAQ,MAAO,CAAE,MAAO,SAAS,KAAK,WAAY,CAAE,CAAE,EACjJ,EAAsB,KAAM,WAAU,aAAa,aAAa,CAAO,EACvE,EAAQ,GAAI,SAAQ,AAAC,GAAY,CAAE,EAAI,MAAM,aAAe,IAAM,EAAQ,EAAI,CAAG,CAAC,EACxF,EAAI,MAAM,UAAY,EACtB,EAAI,MAAM,KAAK,EACf,KAAM,GACN,EAAI,OAAO,MAAQ,EAAI,MAAM,WAC7B,EAAI,OAAO,OAAS,EAAI,MAAM,YAC9B,GAAM,GAA0B,EAAO,eAAe,EAAE,GAClD,EAAgD,EAAM,gBAAkB,EAAM,gBAAgB,EAAI,GAClG,EAAwC,EAAM,YAAc,EAAM,YAAY,EAAI,GAClF,EAA8C,EAAM,eAAiB,EAAM,eAAe,EAAI,GACpG,EAAI,SAAU,EAAI,MAAM,WAAY,EAAI,MAAM,YAAa,EAAM,MAAO,CAAE,SAAQ,QAAO,WAAU,cAAa,cAAa,CAAC,EAC9H,EAAI,OAAO,QAAU,IAAM,CACzB,AAAI,EAAI,MAAM,OAAQ,EAAI,MAAM,KAAK,EAChC,EAAI,MAAM,MAAM,CACvB,CACF,CAEA,kBAA+B,CAC7B,GAAI,CAAC,EAAI,MAAM,OAAQ,CAErB,KAAM,GAAM,OAAO,EAAI,KAAK,EAC5B,GAAM,GAAU,EAAM,GAAG,OAAO,EAAE,WAClC,AAAI,EAAU,EAAU,UAAY,GAAG,EAAI,qBAAsB,EAAU,EAAU,OAAO,EAC5F,EAAU,QAAU,CACtB,CACA,GAAM,GAAM,EAAM,IAAI,EACtB,EAAI,OAAS,IAAQ,GAAM,EAAU,QACrC,EAAU,OAAS,EACnB,sBAAsB,CAAa,CACrC,CAEA,kBAA0B,CACxB,GAAI,CAAC,EAAI,MAAM,OAAQ,CACrB,GAAM,GAAe,KAAM,GAAM,KAAK,EAAM,MAAM,EAClD,KAAM,GAAM,KAAK,OAAO,EAAI,MAAO,EAAI,MAAM,EAC7C,KAAM,GAAM,KAAK,IAAI,EAAI,OAAQ,CAAY,EAC7C,EAAK,EAAa,WAAW,CAC/B,CACA,GAAM,GAAM,EAAM,IAAI,EACtB,EAAI,KAAO,IAAQ,GAAM,EAAU,MACnC,EAAU,KAAO,EACjB,EAAO,EAAI,MAAM,OAAS,SAAW,QAAQ,EAAI,OAAO,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,cAAc,EAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,QAAQ,EAE3I,WAAW,EAAU,EAAE,CACzB,CAEA,kBAAsB,CACpB,EAAI,iBAAkB,EAAM,QAAS,kBAAmB,EAAM,GAAG,QAAQ,YAAY,EACrF,EAAI,YAAa,EAAM,IAAI,SAAU,WAAY,EAAM,IAAI,KAAK,EAChE,EAAO,YAAY,EACnB,KAAM,GAAM,KAAK,EACjB,EAAI,WAAY,EAAM,GAAG,WAAW,EAAG,eAAgB,EAAM,IAAI,QAAQ,EACzE,EAAI,iBAAkB,OAAO,OAAO,EAAM,MAAM,EAAE,OAAO,AAAC,GAAU,IAAU,IAAI,EAAE,MAAM,EAC1F,EAAO,iBAAiB,EACxB,KAAM,GAAM,OAAO,EACnB,KAAM,GAAO,EACb,KAAM,GAAc,EACpB,KAAM,GAAS,CACjB,CAEA,OAAO,OAAS", + "sourcesContent": ["/**\n * Human demo for browsers\n * @default Human Library\n * @summary \n * @author \n * @copyright \n * @license MIT\n */\n\nimport { Human, Config } from '../../dist/human.esm.js'; // equivalent of @vladmandic/Human\n\nconst humanConfig: Partial = { // user configuration for human, used to fine-tune behavior\n // backend: 'webgpu' as const,\n // async: true,\n modelBasePath: '../../models',\n filter: { enabled: true, equalization: false },\n cacheSensitivity: 0,\n face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },\n body: { enabled: true },\n hand: { enabled: true },\n object: { enabled: true },\n gesture: { enabled: true },\n};\n\nconst human = new Human(humanConfig); // create instance of human with overrides from user configuration\n\nhuman.env['perfadd'] = false; // is performance data showing instant or total values\nhuman.draw.options.font = 'small-caps 18px \"Lato\"'; // set font used to draw labels when using draw methods\nhuman.draw.options.lineHeight = 20;\n\nconst dom = { // grab instances of dom objects so we dont have to look them up later\n video: document.getElementById('video') as HTMLVideoElement,\n canvas: document.getElementById('canvas') as HTMLCanvasElement,\n log: document.getElementById('log') as HTMLPreElement,\n fps: document.getElementById('status') as HTMLPreElement,\n perf: document.getElementById('performance') as HTMLDivElement,\n};\nconst timestamp = { detect: 0, draw: 0, tensors: 0 }; // holds information used to calculate performance and possible memory leaks\nconst fps = { detect: 0, draw: 0 }; // holds calculated fps information for both detect and screen refresh\n\nconst log = (...msg) => { // helper method to output messages\n dom.log.innerText += msg.join(' ') + '\\n';\n // eslint-disable-next-line no-console\n console.log(...msg);\n};\nconst status = (msg) => dom.fps.innerText = msg; // print status element\nconst perf = (msg) => dom.perf.innerText = 'tensors:' + human.tf.memory().numTensors + ' | performance: ' + JSON.stringify(msg).replace(/\"|{|}/g, '').replace(/,/g, ' | '); // print performance element\n\nasync function webCam() { // initialize webcam\n status('starting webcam...');\n // @ts-ignore resizeMode is not yet defined in tslib\n const options: MediaStreamConstraints = { audio: false, video: { facingMode: 'user', resizeMode: 'none', width: { ideal: document.body.clientWidth } } };\n const stream: MediaStream = await navigator.mediaDevices.getUserMedia(options);\n const ready = new Promise((resolve) => { dom.video.onloadeddata = () => resolve(true); });\n dom.video.srcObject = stream;\n dom.video.play();\n await ready;\n dom.canvas.width = dom.video.videoWidth;\n dom.canvas.height = dom.video.videoHeight;\n const track: MediaStreamTrack = stream.getVideoTracks()[0];\n const capabilities: MediaTrackCapabilities | string = track.getCapabilities ? track.getCapabilities() : '';\n const settings: MediaTrackSettings | string = track.getSettings ? track.getSettings() : '';\n const constraints: MediaTrackConstraints | string = track.getConstraints ? track.getConstraints() : '';\n log('video:', dom.video.videoWidth, dom.video.videoHeight, track.label, { stream, track, settings, constraints, capabilities });\n dom.canvas.onclick = () => { // pause when clicked on screen and resume on next click\n if (dom.video.paused) dom.video.play();\n else dom.video.pause();\n };\n}\n\nasync function detectionLoop() { // main detection loop\n if (!dom.video.paused) {\n // console.log('profiling data:', await human.profile(dom.video));\n await human.detect(dom.video); // actual detection; were not capturing output in a local variable as it can also be reached via human.result\n const tensors = human.tf.memory().numTensors; // check current tensor usage for memory leaks\n if (tensors - timestamp.tensors !== 0) log('allocated tensors:', tensors - timestamp.tensors); // printed on start and each time there is a tensor leak\n timestamp.tensors = tensors;\n }\n const now = human.now();\n fps.detect = 1000 / (now - timestamp.detect);\n timestamp.detect = now;\n requestAnimationFrame(detectionLoop); // start new frame immediately\n}\n\nasync function drawLoop() { // main screen refresh loop\n if (!dom.video.paused) {\n const interpolated = await human.next(human.result); // smoothen result using last-known results\n await human.draw.canvas(dom.video, dom.canvas); // draw canvas to screen\n await human.draw.all(dom.canvas, interpolated); // draw labels, boxes, lines, etc.\n perf(interpolated.performance); // write performance data\n }\n const now = human.now();\n fps.draw = 1000 / (now - timestamp.draw);\n timestamp.draw = now;\n status(dom.video.paused ? 'paused' : `fps: ${fps.detect.toFixed(1).padStart(5, ' ')} detect | ${fps.draw.toFixed(1).padStart(5, ' ')} draw`); // write status\n // requestAnimationFrame(drawLoop); // refresh at screen refresh rate\n setTimeout(drawLoop, 30); // use to slow down refresh from max refresh rate to target of 30 fps\n}\n\nasync function main() { // main entry point\n log('human version:', human.version, '| tfjs version:', human.tf.version['tfjs-core']);\n log('platform:', human.env.platform, '| agent:', human.env.agent);\n status('loading...');\n await human.load(); // preload all models\n log('backend:', human.tf.getBackend(), '| available:', human.env.backends);\n log('loaded models:', Object.values(human.models).filter((model) => model !== null).length);\n status('initializing...');\n await human.warmup(); // warmup function to initialize backend for future faster detection\n await webCam(); // start webcam\n await detectionLoop(); // start detection loop\n await drawLoop(); // start draw loop\n}\n\nwindow.onload = main;\n"], + "mappings": ";;;;;;AASA,gDATA,AAWA,GAAM,GAA+B,CAGnC,cAAe,eACf,OAAQ,CAAE,QAAS,GAAM,aAAc,EAAM,EAC7C,iBAAkB,EAClB,KAAM,CAAE,QAAS,GAAM,SAAU,CAAE,SAAU,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,UAAW,CAAE,QAAS,EAAM,EAAG,KAAM,CAAE,QAAS,EAAK,EAAG,YAAa,CAAE,QAAS,EAAK,EAAG,QAAS,CAAE,QAAS,EAAK,CAAE,EAClM,KAAM,CAAE,QAAS,EAAK,EACtB,KAAM,CAAE,QAAS,EAAK,EACtB,OAAQ,CAAE,QAAS,EAAK,EACxB,QAAS,CAAE,QAAS,EAAK,CAC3B,EAEM,EAAQ,GAAI,GAAM,CAAW,EAEnC,EAAM,IAAI,QAAa,GACvB,EAAM,KAAK,QAAQ,KAAO,yBAC1B,EAAM,KAAK,QAAQ,WAAa,GAEhC,GAAM,GAAM,CACV,MAAO,SAAS,eAAe,OAAO,EACtC,OAAQ,SAAS,eAAe,QAAQ,EACxC,IAAK,SAAS,eAAe,KAAK,EAClC,IAAK,SAAS,eAAe,QAAQ,EACrC,KAAM,SAAS,eAAe,aAAa,CAC7C,EACM,EAAY,CAAE,OAAQ,EAAG,KAAM,EAAG,QAAS,CAAE,EAC7C,EAAM,CAAE,OAAQ,EAAG,KAAM,CAAE,EAE3B,EAAM,IAAI,IAAQ,CACtB,EAAI,IAAI,WAAa,EAAI,KAAK,GAAG,EAAI;AAAA,EAErC,QAAQ,IAAI,GAAG,CAAG,CACpB,EACM,EAAS,AAAC,GAAQ,EAAI,IAAI,UAAY,EACtC,EAAO,AAAC,GAAQ,EAAI,KAAK,UAAY,WAAa,EAAM,GAAG,OAAO,EAAE,WAAa,mBAAqB,KAAK,UAAU,CAAG,EAAE,QAAQ,SAAU,EAAE,EAAE,QAAQ,KAAM,KAAK,EAEzK,kBAAwB,CACtB,EAAO,oBAAoB,EAE3B,GAAM,GAAkC,CAAE,MAAO,GAAO,MAAO,CAAE,WAAY,OAAQ,WAAY,OAAQ,MAAO,CAAE,MAAO,SAAS,KAAK,WAAY,CAAE,CAAE,EACjJ,EAAsB,KAAM,WAAU,aAAa,aAAa,CAAO,EACvE,EAAQ,GAAI,SAAQ,AAAC,GAAY,CAAE,EAAI,MAAM,aAAe,IAAM,EAAQ,EAAI,CAAG,CAAC,EACxF,EAAI,MAAM,UAAY,EACtB,EAAI,MAAM,KAAK,EACf,KAAM,GACN,EAAI,OAAO,MAAQ,EAAI,MAAM,WAC7B,EAAI,OAAO,OAAS,EAAI,MAAM,YAC9B,GAAM,GAA0B,EAAO,eAAe,EAAE,GAClD,EAAgD,EAAM,gBAAkB,EAAM,gBAAgB,EAAI,GAClG,EAAwC,EAAM,YAAc,EAAM,YAAY,EAAI,GAClF,EAA8C,EAAM,eAAiB,EAAM,eAAe,EAAI,GACpG,EAAI,SAAU,EAAI,MAAM,WAAY,EAAI,MAAM,YAAa,EAAM,MAAO,CAAE,SAAQ,QAAO,WAAU,cAAa,cAAa,CAAC,EAC9H,EAAI,OAAO,QAAU,IAAM,CACzB,AAAI,EAAI,MAAM,OAAQ,EAAI,MAAM,KAAK,EAChC,EAAI,MAAM,MAAM,CACvB,CACF,CAEA,kBAA+B,CAC7B,GAAI,CAAC,EAAI,MAAM,OAAQ,CAErB,KAAM,GAAM,OAAO,EAAI,KAAK,EAC5B,GAAM,GAAU,EAAM,GAAG,OAAO,EAAE,WAClC,AAAI,EAAU,EAAU,UAAY,GAAG,EAAI,qBAAsB,EAAU,EAAU,OAAO,EAC5F,EAAU,QAAU,CACtB,CACA,GAAM,GAAM,EAAM,IAAI,EACtB,EAAI,OAAS,IAAQ,GAAM,EAAU,QACrC,EAAU,OAAS,EACnB,sBAAsB,CAAa,CACrC,CAEA,kBAA0B,CACxB,GAAI,CAAC,EAAI,MAAM,OAAQ,CACrB,GAAM,GAAe,KAAM,GAAM,KAAK,EAAM,MAAM,EAClD,KAAM,GAAM,KAAK,OAAO,EAAI,MAAO,EAAI,MAAM,EAC7C,KAAM,GAAM,KAAK,IAAI,EAAI,OAAQ,CAAY,EAC7C,EAAK,EAAa,WAAW,CAC/B,CACA,GAAM,GAAM,EAAM,IAAI,EACtB,EAAI,KAAO,IAAQ,GAAM,EAAU,MACnC,EAAU,KAAO,EACjB,EAAO,EAAI,MAAM,OAAS,SAAW,QAAQ,EAAI,OAAO,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,cAAc,EAAI,KAAK,QAAQ,CAAC,EAAE,SAAS,EAAG,GAAG,QAAQ,EAE3I,WAAW,EAAU,EAAE,CACzB,CAEA,kBAAsB,CACpB,EAAI,iBAAkB,EAAM,QAAS,kBAAmB,EAAM,GAAG,QAAQ,YAAY,EACrF,EAAI,YAAa,EAAM,IAAI,SAAU,WAAY,EAAM,IAAI,KAAK,EAChE,EAAO,YAAY,EACnB,KAAM,GAAM,KAAK,EACjB,EAAI,WAAY,EAAM,GAAG,WAAW,EAAG,eAAgB,EAAM,IAAI,QAAQ,EACzE,EAAI,iBAAkB,OAAO,OAAO,EAAM,MAAM,EAAE,OAAO,AAAC,GAAU,IAAU,IAAI,EAAE,MAAM,EAC1F,EAAO,iBAAiB,EACxB,KAAM,GAAM,OAAO,EACnB,KAAM,GAAO,EACb,KAAM,GAAc,EACpB,KAAM,GAAS,CACjB,CAEA,OAAO,OAAS", "names": [] } diff --git a/demo/typescript/index.ts b/demo/typescript/index.ts index 25c77d9b..1e0a3406 100644 --- a/demo/typescript/index.ts +++ b/demo/typescript/index.ts @@ -14,7 +14,8 @@ const humanConfig: Partial = { // user configuration for human, used to // async: true, modelBasePath: '../../models', filter: { enabled: true, equalization: false }, - face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } }, + cacheSensitivity: 0, + face: { enabled: true, detector: { rotation: false }, mesh: { enabled: true }, attention: { enabled: false }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } }, body: { enabled: true }, hand: { enabled: true }, object: { enabled: true }, diff --git a/package.json b/package.json index cf945e6d..07d7d345 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vladmandic/human", - "version": "2.6.5", + "version": "2.7.0", "description": "Human: AI-powered 3D Face Detection & Rotation Tracking, Face Description & Recognition, Body Pose Tracking, 3D Hand & Finger Tracking, Iris Analysis, Age & Gender & Emotion Prediction, Gesture Recognition", "sideEffects": false, "main": "dist/human.node.js", @@ -86,7 +86,7 @@ "rimraf": "^3.0.2", "seedrandom": "^3.0.5", "tslib": "^2.3.1", - "typedoc": "0.22.14", + "typedoc": "0.22.15", "typescript": "4.6.3" } } diff --git a/src/config.ts b/src/config.ts index d1a004e4..f2626d99 100644 --- a/src/config.ts +++ b/src/config.ts @@ -40,6 +40,9 @@ export interface FaceMeshConfig extends GenericConfig {} /** Iris part of face configuration */ export interface FaceIrisConfig extends GenericConfig {} +/** Attention part of face configuration */ +export interface FaceAttentionConfig extends GenericConfig {} + /** Description or face embedding part of face configuration * - also used by age and gender detection */ @@ -64,6 +67,7 @@ export interface FaceLivenessConfig extends GenericConfig {} export interface FaceConfig extends GenericConfig { detector: Partial, mesh: Partial, + attention: Partial, iris: Partial, description: Partial, emotion: Partial, @@ -349,6 +353,10 @@ const config: Config = { enabled: true, modelPath: 'facemesh.json', }, + attention: { + enabled: false, + modelPath: 'facemesh-attention.json', + }, iris: { enabled: true, modelPath: 'iris.json', diff --git a/src/draw/body.ts b/src/draw/body.ts new file mode 100644 index 00000000..6ff6d342 --- /dev/null +++ b/src/draw/body.ts @@ -0,0 +1,51 @@ +import { mergeDeep } from '../util/util'; +import { getCanvasContext, rect, point, curves, colorDepth } from './primitives'; +import { options } from './options'; +import type { BodyResult } from '../result'; +import type { AnyCanvas, DrawOptions } from '../exports'; + +/** draw detected bodies */ +export async function body(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + ctx.lineJoin = 'round'; + for (let i = 0; i < result.length; i++) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + ctx.lineWidth = localOptions.lineWidth; + ctx.font = localOptions.font; + if (localOptions.drawBoxes && result[i].box && result[i].box?.length === 4) { + rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions); + if (localOptions.drawLabels) { + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(`body ${100 * result[i].score}%`, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + } + if (localOptions.drawPoints && result[i].keypoints) { + for (let pt = 0; pt < result[i].keypoints.length; pt++) { + if (!result[i].keypoints[pt].score || (result[i].keypoints[pt].score === 0)) continue; + ctx.fillStyle = localOptions.useDepth && result[i].keypoints[pt].position[2] ? colorDepth(result[i].keypoints[pt].position[2] || 0) : localOptions.color; + point(ctx, result[i].keypoints[pt].position[0], result[i].keypoints[pt].position[1], 0, localOptions); + } + } + if (localOptions.drawLabels && result[i].keypoints) { + ctx.font = localOptions.font; + for (const pt of result[i].keypoints) { + if (!pt.score || (pt.score === 0)) continue; + ctx.fillStyle = localOptions.useDepth && pt.position[2] ? colorDepth(pt.position[2]) : localOptions.color; + ctx.fillText(`${pt.part} ${Math.trunc(100 * pt.score)}%`, pt.position[0] + 4, pt.position[1] + 4); + } + } + if (localOptions.drawPolygons && result[i].keypoints && result[i].annotations) { + for (const part of Object.values(result[i].annotations)) { + for (const connected of part) curves(ctx, connected, localOptions); + } + } + } +} diff --git a/src/draw/draw.ts b/src/draw/draw.ts new file mode 100644 index 00000000..5b519999 --- /dev/null +++ b/src/draw/draw.ts @@ -0,0 +1,78 @@ +/** + * Module that implements helper draw functions, exposed as human.draw + */ + +import { mergeDeep, now } from '../util/util'; +import { env } from '../util/env'; +import { getCanvasContext, rect } from './primitives'; +import { options } from './options'; +import { face } from './face'; +import { body } from './body'; +import { hand } from './hand'; +import { object } from './object'; +import { gesture } from './gesture'; +import type { Result, PersonResult } from '../result'; +import type { AnyCanvas, DrawOptions } from '../exports'; + +let drawTime = 0; + +export { options } from './options'; +export { face } from './face'; +export { body } from './body'; +export { hand } from './hand'; +export { object } from './object'; +export { gesture } from './gesture'; + +/** draw combined person results instead of individual detection result objects */ +export async function person(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + ctx.lineJoin = 'round'; + ctx.font = localOptions.font; + + for (let i = 0; i < result.length; i++) { + if (localOptions.drawBoxes) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + rect(ctx, result[i].box[0], result[i].box[1], result[i].box[2], result[i].box[3], localOptions); + if (localOptions.drawLabels) { + const label = `person #${i}`; + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(label, result[i].box[0] + 3, 1 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(label, result[i].box[0] + 2, 0 + result[i].box[1] + localOptions.lineHeight, result[i].box[2]); + } + ctx.stroke(); + } + } +} + +/** draw processed canvas */ +export async function canvas(input: AnyCanvas | HTMLImageElement | HTMLVideoElement, output: AnyCanvas) { + if (!input || !output) return; + const ctx = getCanvasContext(output); + if (!ctx) return; + ctx.drawImage(input, 0, 0); +} + +/** meta-function that performs draw for: canvas, face, body, hand */ +export async function all(inCanvas: AnyCanvas, result: Result, drawOptions?: Partial) { + if (!result || !result.performance || !result || !inCanvas) return null; + const timeStamp = now(); + const localOptions = mergeDeep(options, drawOptions); + const promise = Promise.all([ + face(inCanvas, result.face, localOptions), + body(inCanvas, result.body, localOptions), + hand(inCanvas, result.hand, localOptions), + object(inCanvas, result.object, localOptions), + gesture(inCanvas, result.gesture, localOptions), // gestures do not have buffering + // person(inCanvas, result.persons, localOptions); // already included above + ]); + drawTime = env.perfadd ? drawTime + Math.round(now() - timeStamp) : Math.round(now() - timeStamp); + result.performance.draw = drawTime; + return promise; +} diff --git a/src/draw/face.ts b/src/draw/face.ts new file mode 100644 index 00000000..00277a83 --- /dev/null +++ b/src/draw/face.ts @@ -0,0 +1,134 @@ +import { TRI468 as triangulation } from '../face/facemeshcoords'; +import { mergeDeep } from '../util/util'; +import { getCanvasContext, rad2deg, rect, point, lines, arrow } from './primitives'; +import { options } from './options'; +import type { FaceResult } from '../result'; +import type { AnyCanvas, DrawOptions } from '../exports'; + +/** draw detected faces */ +export async function face(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + for (const f of result) { + ctx.font = localOptions.font; + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + if (localOptions.drawBoxes) rect(ctx, f.box[0], f.box[1], f.box[2], f.box[3], localOptions); + if (localOptions.drawLabels) { + // silly hack since fillText does not suport new line + const labels:string[] = []; + labels.push(`face: ${Math.trunc(100 * f.score)}%`); + if (f.genderScore) labels.push(`${f.gender || ''} ${Math.trunc(100 * f.genderScore)}%`); + if (f.age) labels.push(`age: ${f.age || ''}`); + if (f.iris) labels.push(`distance: ${f.iris}`); + if (f.real) labels.push(`real: ${Math.trunc(100 * f.real)}%`); + if (f.live) labels.push(`live: ${Math.trunc(100 * f.live)}%`); + if (f.emotion && f.emotion.length > 0) { + const emotion = f.emotion.map((a) => `${Math.trunc(100 * a.score)}% ${a.emotion}`); + if (emotion.length > 3) emotion.length = 3; + labels.push(emotion.join(' ')); + } + if (f.rotation && f.rotation.angle && f.rotation.gaze) { + if (f.rotation.angle.roll) labels.push(`roll: ${rad2deg(f.rotation.angle.roll)}° yaw:${rad2deg(f.rotation.angle.yaw)}° pitch:${rad2deg(f.rotation.angle.pitch)}°`); + if (f.rotation.gaze.bearing) labels.push(`gaze: ${rad2deg(f.rotation.gaze.bearing)}°`); + } + if (labels.length === 0) labels.push('face'); + ctx.fillStyle = localOptions.color; + for (let i = labels.length - 1; i >= 0; i--) { + const x = Math.max(f.box[0], 0); + const y = i * localOptions.lineHeight + f.box[1]; + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(labels[i], x + 5, y + 16); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(labels[i], x + 4, y + 15); + } + } + // ctx.lineWidth = localOptions.lineWidth; + ctx.lineWidth = 2; + if (f.mesh && f.mesh.length > 0) { + if (localOptions.drawPoints) { + const length = Math.max(468, f.mesh.length); + for (let i = 0; i < length; i++) point(ctx, f.mesh[i][0], f.mesh[i][1], f.mesh[i][2], localOptions); + } + if (localOptions.drawAttention && f.mesh.length > 468) { + for (let i = 468; i < f.mesh.length; i++) point(ctx, f.mesh[i][0], f.mesh[i][1], -255, localOptions); + } + if (localOptions.drawPolygons) { + if (f.mesh.length > 450) { + for (let i = 0; i < triangulation.length / 3; i++) { + const points = [ + triangulation[i * 3 + 0], + triangulation[i * 3 + 1], + triangulation[i * 3 + 2], + ].map((index) => f.mesh[index]); + lines(ctx, points, localOptions); + } + } + // iris: array[center, left, top, right, bottom] + if (f.annotations && f.annotations['leftEyeIris'] && f.annotations['leftEyeIris'][0]) { + ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; + ctx.beginPath(); + const sizeX = Math.abs(f.annotations['leftEyeIris'][3][0] - f.annotations['leftEyeIris'][1][0]) / 2; + const sizeY = Math.abs(f.annotations['leftEyeIris'][4][1] - f.annotations['leftEyeIris'][2][1]) / 2; + ctx.ellipse(f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + ctx.stroke(); + if (localOptions.fillPolygons) { + ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; + ctx.fill(); + } + } + if (f.annotations && f.annotations['rightEyeIris'] && f.annotations['rightEyeIris'][0]) { + ctx.strokeStyle = localOptions.useDepth ? 'rgba(255, 200, 255, 0.3)' : localOptions.color; + ctx.beginPath(); + const sizeX = Math.abs(f.annotations['rightEyeIris'][3][0] - f.annotations['rightEyeIris'][1][0]) / 2; + const sizeY = Math.abs(f.annotations['rightEyeIris'][4][1] - f.annotations['rightEyeIris'][2][1]) / 2; + ctx.ellipse(f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1], sizeX, sizeY, 0, 0, 2 * Math.PI); + ctx.stroke(); + if (localOptions.fillPolygons) { + ctx.fillStyle = localOptions.useDepth ? 'rgba(255, 255, 200, 0.3)' : localOptions.color; + ctx.fill(); + } + } + if (localOptions.drawGaze && f.rotation?.angle && typeof Path2D !== 'undefined') { + ctx.strokeStyle = 'pink'; + const valX = (f.box[0] + f.box[2] / 2) - (f.box[3] * rad2deg(f.rotation.angle.yaw) / 90); + const valY = (f.box[1] + f.box[3] / 2) + (f.box[2] * rad2deg(f.rotation.angle.pitch) / 90); + const pathV = new Path2D(` + M ${f.box[0] + f.box[2] / 2} ${f.box[1]} + C + ${valX} ${f.box[1]}, + ${valX} ${f.box[1] + f.box[3]}, + ${f.box[0] + f.box[2] / 2} ${f.box[1] + f.box[3]} + `); + const pathH = new Path2D(` + M ${f.box[0]} ${f.box[1] + f.box[3] / 2} + C + ${f.box[0]} ${valY}, + ${f.box[0] + f.box[2]} ${valY}, + ${f.box[0] + f.box[2]} ${f.box[1] + f.box[3] / 2} + `); + ctx.stroke(pathH); + ctx.stroke(pathV); + } + if (localOptions.drawGaze && f.rotation?.gaze?.strength && f.rotation?.gaze?.bearing && f.annotations['leftEyeIris'] && f.annotations['rightEyeIris'] && f.annotations['leftEyeIris'][0] && f.annotations['rightEyeIris'][0]) { + ctx.strokeStyle = 'pink'; + ctx.fillStyle = 'pink'; + const leftGaze = [ + f.annotations['leftEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]), + f.annotations['leftEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]), + ]; + arrow(ctx, [f.annotations['leftEyeIris'][0][0], f.annotations['leftEyeIris'][0][1]], [leftGaze[0], leftGaze[1]], 4); + const rightGaze = [ + f.annotations['rightEyeIris'][0][0] + (Math.sin(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[3]), + f.annotations['rightEyeIris'][0][1] + (Math.cos(f.rotation.gaze.bearing) * f.rotation.gaze.strength * f.box[2]), + ]; + arrow(ctx, [f.annotations['rightEyeIris'][0][0], f.annotations['rightEyeIris'][0][1]], [rightGaze[0], rightGaze[1]], 4); + } + } + } + } +} diff --git a/src/draw/gesture.ts b/src/draw/gesture.ts new file mode 100644 index 00000000..a7dc7682 --- /dev/null +++ b/src/draw/gesture.ts @@ -0,0 +1,34 @@ +import { mergeDeep } from '../util/util'; +import { getCanvasContext } from './primitives'; +import { options } from './options'; +import type { GestureResult } from '../result'; +import type { AnyCanvas, DrawOptions } from '../exports'; + +/** draw detected gestures */ +export async function gesture(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + if (localOptions.drawGestures) { + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + ctx.font = localOptions.font; + ctx.fillStyle = localOptions.color; + let i = 1; + for (let j = 0; j < result.length; j++) { + let where: unknown[] = []; // what&where is a record + let what: unknown[] = []; // what&where is a record + [where, what] = Object.entries(result[j]); + if ((what.length > 1) && ((what[1] as string).length > 0)) { + const who = where[1] as number > 0 ? `#${where[1]}` : ''; + const label = `${where[0]} ${who}: ${what[1]}`; + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(label, 8, 2 + (i * localOptions.lineHeight)); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(label, 6, 0 + (i * localOptions.lineHeight)); + i += 1; + } + } + } +} diff --git a/src/draw/hand.ts b/src/draw/hand.ts new file mode 100644 index 00000000..50474443 --- /dev/null +++ b/src/draw/hand.ts @@ -0,0 +1,74 @@ +import { mergeDeep } from '../util/util'; +import { getCanvasContext, rect, point, colorDepth } from './primitives'; +import { options } from './options'; +import type { HandResult } from '../result'; +import type { AnyCanvas, DrawOptions, Point } from '../exports'; + +/** draw detected hands */ +export async function hand(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + ctx.lineJoin = 'round'; + ctx.font = localOptions.font; + for (const h of result) { + if (localOptions.drawBoxes) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions); + if (localOptions.drawLabels) { + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(`hand:${Math.trunc(100 * h.score)}%`, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); // can use h.label + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(`hand:${Math.trunc(100 * h.score)}%`, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); // can use h.label + } + ctx.stroke(); + } + if (localOptions.drawPoints) { + if (h.keypoints && h.keypoints.length > 0) { + for (const pt of h.keypoints) { + ctx.fillStyle = localOptions.useDepth ? colorDepth(pt[2] || 0) : localOptions.color; + point(ctx, pt[0], pt[1], 0, localOptions); + } + } + } + if (localOptions.drawLabels && h.annotations) { + const addHandLabel = (part: Array, title: string) => { + if (!part || part.length === 0 || !part[0]) return; + const z = part[part.length - 1][2] || 0; + ctx.fillStyle = localOptions.useDepth ? colorDepth(z) : localOptions.color; + ctx.fillText(title, part[part.length - 1][0] + 4, part[part.length - 1][1] + 4); + }; + ctx.font = localOptions.font; + addHandLabel(h.annotations['index'], 'index'); + addHandLabel(h.annotations['middle'], 'middle'); + addHandLabel(h.annotations['ring'], 'ring'); + addHandLabel(h.annotations['pinky'], 'pinky'); + addHandLabel(h.annotations['thumb'], 'thumb'); + addHandLabel(h.annotations['palm'], 'palm'); + } + if (localOptions.drawPolygons && h.annotations) { + const addHandLine = (part: Array) => { + if (!part || part.length === 0 || !part[0]) return; + for (let i = 0; i < part.length; i++) { + ctx.beginPath(); + const z = part[i][2] || 0; + ctx.strokeStyle = localOptions.useDepth ? colorDepth(i * z) : localOptions.color; + ctx.moveTo(part[i > 0 ? i - 1 : 0][0], part[i > 0 ? i - 1 : 0][1]); + ctx.lineTo(part[i][0], part[i][1]); + ctx.stroke(); + } + }; + ctx.lineWidth = localOptions.lineWidth; + addHandLine(h.annotations['index']); + addHandLine(h.annotations['middle']); + addHandLine(h.annotations['ring']); + addHandLine(h.annotations['pinky']); + addHandLine(h.annotations['thumb']); + // addPart(h.annotations.palm); + } + } +} diff --git a/src/draw/object.ts b/src/draw/object.ts new file mode 100644 index 00000000..8664e083 --- /dev/null +++ b/src/draw/object.ts @@ -0,0 +1,32 @@ +import { mergeDeep } from '../util/util'; +import { getCanvasContext, rect } from './primitives'; +import { options } from './options'; +import type { ObjectResult } from '../result'; +import type { AnyCanvas, DrawOptions } from '../exports'; + +/** draw detected objects */ +export async function object(inCanvas: AnyCanvas, result: Array, drawOptions?: Partial) { + const localOptions = mergeDeep(options, drawOptions); + if (!result || !inCanvas) return; + const ctx = getCanvasContext(inCanvas); + if (!ctx) return; + ctx.lineJoin = 'round'; + ctx.font = localOptions.font; + for (const h of result) { + if (localOptions.drawBoxes) { + ctx.strokeStyle = localOptions.color; + ctx.fillStyle = localOptions.color; + rect(ctx, h.box[0], h.box[1], h.box[2], h.box[3], localOptions); + if (localOptions.drawLabels) { + const label = `${h.label} ${Math.round(100 * h.score)}%`; + if (localOptions.shadowColor && localOptions.shadowColor !== '') { + ctx.fillStyle = localOptions.shadowColor; + ctx.fillText(label, h.box[0] + 3, 1 + h.box[1] + localOptions.lineHeight, h.box[2]); + } + ctx.fillStyle = localOptions.labelColor; + ctx.fillText(label, h.box[0] + 2, 0 + h.box[1] + localOptions.lineHeight, h.box[2]); + } + ctx.stroke(); + } + } +} diff --git a/src/draw/options.ts b/src/draw/options.ts new file mode 100644 index 00000000..79cde324 --- /dev/null +++ b/src/draw/options.ts @@ -0,0 +1,66 @@ +/** Draw Options + * - Accessed via `human.draw.options` or provided per each draw method as the drawOptions optional parameter + */ +export type DrawOptions = { + /** draw line color */ + color: string, + /** alpha value used for lines */ + alpha: number, + /** label color */ + labelColor: string, + /** label shadow color */ + shadowColor: string, + /** label font */ + font: string, + /** line spacing between labels */ + lineHeight: number, + /** line width for drawn lines */ + lineWidth: number, + /** size of drawn points */ + pointSize: number, + /** draw rounded boxes by n pixels */ + roundRect: number, + /** should points be drawn? */ + drawPoints: boolean, + /** should labels be drawn? */ + drawLabels: boolean, + /** should face attention keypoints be highlighted */ + drawAttention: boolean; + /** should detected gestures be drawn? */ + drawGestures: boolean, + /** should draw boxes around detection results? */ + drawBoxes: boolean, + /** should draw polygons from detection points? */ + drawPolygons: boolean, + /** should draw gaze arrows? */ + drawGaze: boolean, + /** should fill polygons? */ + fillPolygons: boolean, + /** use z-coordinate when available */ + useDepth: boolean, + /** should lines be curved? */ + useCurves: boolean, +} + +/** currently set draw options {@link DrawOptions} */ +export const options: DrawOptions = { + color: 'rgba(173, 216, 230, 0.6)', // 'lightblue' with light alpha channel + labelColor: 'rgba(173, 216, 230, 1)', // 'lightblue' with dark alpha channel + shadowColor: 'black', + alpha: 0.5, + font: 'small-caps 16px "Segoe UI"', + lineHeight: 18, + lineWidth: 4, + pointSize: 2, + roundRect: 8, + drawPoints: false, + drawLabels: true, + drawBoxes: true, + drawAttention: true, + drawGestures: true, + drawPolygons: true, + drawGaze: true, + fillPolygons: false, + useDepth: true, + useCurves: false, +}; diff --git a/src/draw/primitives.ts b/src/draw/primitives.ts new file mode 100644 index 00000000..a6bf282d --- /dev/null +++ b/src/draw/primitives.ts @@ -0,0 +1,115 @@ +import { log } from '../util/util'; +import type { AnyCanvas } from '../exports'; +import type { Point } from '../result'; +import { options, DrawOptions } from './options'; + +export const getCanvasContext = (input: AnyCanvas) => { + if (!input) log('draw error: invalid canvas'); + else if (!input.getContext) log('draw error: canvas context not defined'); + else { + const ctx = input.getContext('2d'); + if (!ctx) log('draw error: cannot get canvas context'); + else return ctx; + } + return null; +}; + +export const rad2deg = (theta: number) => Math.round((theta * 180) / Math.PI); +export const colorDepth = (z: number, rgb: [boolean, boolean, boolean] = [true, true, false]): string => { + const r = rgb[0] ? 127 + Math.trunc(3 * z) : 255; + const g = rgb[1] ? 127 - Math.trunc(3 * z) : 255; + const b = rgb[2] ? 127 - Math.trunc(3 * z) : 255; + return `rgba(${r}, ${g}, ${b}, ${options.alpha})`; +}; + +export function point(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, x: number, y: number, z: number | undefined, localOptions: DrawOptions) { + z = z || 0; + ctx.fillStyle = localOptions.useDepth && z ? colorDepth(z, z === -255 ? [true, false, true] : [true, false, false]) : localOptions.color; + ctx.beginPath(); + ctx.arc(x, y, localOptions.pointSize, 0, 2 * Math.PI); + ctx.fill(); +} + +export function rect(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, x: number, y: number, width: number, height: number, localOptions: DrawOptions) { + ctx.beginPath(); + ctx.lineWidth = localOptions.lineWidth; + if (localOptions.useCurves) { + const cx = (x + x + width) / 2; + const cy = (y + y + height) / 2; + ctx.ellipse(cx, cy, width / 2, height / 2, 0, 0, 2 * Math.PI); + } else { + ctx.moveTo(x + localOptions.roundRect, y); + ctx.lineTo(x + width - localOptions.roundRect, y); + ctx.quadraticCurveTo(x + width, y, x + width, y + localOptions.roundRect); + ctx.lineTo(x + width, y + height - localOptions.roundRect); + ctx.quadraticCurveTo(x + width, y + height, x + width - localOptions.roundRect, y + height); + ctx.lineTo(x + localOptions.roundRect, y + height); + ctx.quadraticCurveTo(x, y + height, x, y + height - localOptions.roundRect); + ctx.lineTo(x, y + localOptions.roundRect); + ctx.quadraticCurveTo(x, y, x + localOptions.roundRect, y); + ctx.closePath(); + } + ctx.stroke(); +} + +export function lines(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, points: Point[], localOptions: DrawOptions) { + if (points.length < 2) return; + ctx.beginPath(); + ctx.moveTo(points[0][0], points[0][1]); + for (const pt of points) { + const z = pt[2] || 0; + ctx.strokeStyle = localOptions.useDepth && z !== 0 ? colorDepth(z) : localOptions.color; + ctx.fillStyle = localOptions.useDepth && z !== 0 ? colorDepth(z) : localOptions.color; + ctx.lineTo(pt[0], Math.round(pt[1])); + } + ctx.stroke(); + if (localOptions.fillPolygons) { + ctx.closePath(); + ctx.fill(); + } +} + +export function curves(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, points: Point[], localOptions: DrawOptions) { + if (points.length < 2) return; + ctx.lineWidth = localOptions.lineWidth; + if (!localOptions.useCurves || points.length <= 2) { + lines(ctx, points, localOptions); + return; + } + ctx.moveTo(points[0][0], points[0][1]); + for (let i = 0; i < points.length - 2; i++) { + const xc = (points[i][0] + points[i + 1][0]) / 2; + const yc = (points[i][1] + points[i + 1][1]) / 2; + ctx.quadraticCurveTo(points[i][0], points[i][1], xc, yc); + } + ctx.quadraticCurveTo(points[points.length - 2][0], points[points.length - 2][1], points[points.length - 1][0], points[points.length - 1][1]); + ctx.stroke(); + if (localOptions.fillPolygons) { + ctx.closePath(); + ctx.fill(); + } +} + +export function arrow(ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, from: Point, to: Point, radius = 5) { + let angle; + let x; + let y; + ctx.beginPath(); + ctx.moveTo(from[0], from[1]); + ctx.lineTo(to[0], to[1]); + angle = Math.atan2(to[1] - from[1], to[0] - from[0]); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.moveTo(x, y); + angle += (1.0 / 3.0) * (2 * Math.PI); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.lineTo(x, y); + angle += (1.0 / 3.0) * (2 * Math.PI); + x = radius * Math.cos(angle) + to[0]; + y = radius * Math.sin(angle) + to[1]; + ctx.lineTo(x, y); + ctx.closePath(); + ctx.stroke(); + ctx.fill(); +} diff --git a/src/exports.ts b/src/exports.ts index 725624b0..2c242b52 100644 --- a/src/exports.ts +++ b/src/exports.ts @@ -9,7 +9,7 @@ export * from './result'; /* Explict reexport of main @tensorflow/tfjs types */ export type { Tensor, TensorLike, GraphModel, Rank } from './tfjs/types'; -export type { DrawOptions } from './util/draw'; +export type { DrawOptions } from './draw/options'; export type { Descriptor } from './face/match'; export type { Box, Point } from './result'; export type { Models } from './models'; diff --git a/src/face/attention.ts b/src/face/attention.ts new file mode 100644 index 00000000..1654388a --- /dev/null +++ b/src/face/attention.ts @@ -0,0 +1,19 @@ +import type { Tensor } from '../tfjs/types'; + +export async function augment(rawCoords, results: Tensor[]) { + const t: Record = { + eyeL: results[0].dataSync() as Float32Array, // 71 x 2d // output_left_eye + eyeR: results[6].dataSync() as Float32Array, // 71 x 2d // output_right_eye + irisL: results[3].dataSync() as Float32Array, // 5 x 2d // output_left_iris + irisR: results[1].dataSync() as Float32Array, // 5 x 2d // output_right_iris + lips: results[5].dataSync() as Float32Array, // 80 x 2d // output_lips + // flag: results[4], // already processed in parent // conv_faceflag + // mesh: results[2], // already have it in rawCoords // output_mesh_identity + }; + for (let i = 0; i < t.lips.length / 2; i++) rawCoords.push([t.lips[2 * i + 0], t.lips[2 * i + 1], 0]); + for (let i = 0; i < t.eyeL.length / 2; i++) rawCoords.push([t.eyeL[2 * i + 0], t.eyeL[2 * i + 1], 0]); + for (let i = 0; i < t.eyeR.length / 2; i++) rawCoords.push([t.eyeR[2 * i + 0], t.eyeR[2 * i + 1], 0]); + for (let i = 0; i < t.irisL.length / 2; i++) rawCoords.push([t.irisL[2 * i + 0], t.irisL[2 * i + 1], 0]); + for (let i = 0; i < t.irisR.length / 2; i++) rawCoords.push([t.irisR[2 * i + 0], t.irisR[2 * i + 1], 0]); + return rawCoords; +} diff --git a/src/face/blazeface.ts b/src/face/blazeface.ts index 871a4538..88426cc3 100644 --- a/src/face/blazeface.ts +++ b/src/face/blazeface.ts @@ -14,7 +14,7 @@ import { env } from '../util/env'; import type { Point } from '../result'; const keypointsCount = 6; -const faceBoxScaleFactor = 1.2; +const faceBoxScaleFactor = 1.4; let model: GraphModel | null; let anchors: Tensor | null = null; let inputSize = 0; diff --git a/src/face/facemesh.ts b/src/face/facemesh.ts index 07fd63d9..2dbb8670 100644 --- a/src/face/facemesh.ts +++ b/src/face/facemesh.ts @@ -14,6 +14,7 @@ import * as blazeface from './blazeface'; import * as util from './facemeshutil'; import * as coords from './facemeshcoords'; import * as iris from './iris'; +import * as attention from './attention'; import { histogramEqualization } from '../image/enhance'; import { env } from '../util/env'; import type { GraphModel, Tensor } from '../tfjs/types'; @@ -78,20 +79,27 @@ export async function predict(input: Tensor, config: Config): Promise [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]); - for (const key of Object.keys(coords.blazeFaceLandmarks)) face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations + for (const key of Object.keys(coords.blazeFaceLandmarks)) { + face.annotations[key] = [face.mesh[coords.blazeFaceLandmarks[key] as number]]; // add annotations + } } else if (!model) { // mesh enabled, but not loaded if (config.debug) log('face mesh detection requested, but model is not loaded'); } else { // mesh enabled - const [contours, confidence, contourCoords] = model.execute(face.tensor as Tensor) as Array; // first returned tensor represents facial contours which are already included in the coordinates. + const results = model.execute(face.tensor as Tensor) as Array; + const confidence = results.find((t) => t.shape[t.shape.length - 1] === 1) as Tensor; + const contourCoords = results.find((t) => t.shape[t.shape.length - 1] === 1404) as Tensor; const faceConfidence = await confidence.data(); face.faceScore = Math.round(100 * faceConfidence[0]) / 100; const coordsReshaped = tf.reshape(contourCoords, [-1, 3]); let rawCoords = await coordsReshaped.array(); - tf.dispose([contourCoords, coordsReshaped, confidence, contours]); if (face.faceScore < (config.face.detector?.minConfidence || 1)) { // low confidence in detected mesh box.confidence = face.faceScore; // reset confidence of cached box } else { - if (config.face.iris?.enabled) rawCoords = await iris.augmentIris(rawCoords, face.tensor, config, inputSize); // augment results with iris + if (config.face.attention?.enabled) { + rawCoords = await attention.augment(rawCoords, results); // augment iris results using attention model results + } else if (config.face.iris?.enabled) { + rawCoords = await iris.augmentIris(rawCoords, face.tensor, config, inputSize); // run iris model and augment results + } face.mesh = util.transformRawCoords(rawCoords, box, angle, rotationMatrix, inputSize); // get processed mesh face.meshRaw = face.mesh.map((pt) => [pt[0] / (input.shape[2] || 0), pt[1] / (input.shape[1] || 0), (pt[2] || 0) / inputSize]); for (const key of Object.keys(coords.meshAnnotations)) face.annotations[key] = coords.meshAnnotations[key].map((index) => face.mesh[index]); // add annotations @@ -101,6 +109,7 @@ export async function predict(input: Tensor, config: Config): Promise (config.face.detector?.minConfidence || 1)) faces.push(face); else tf.dispose(face.tensor); @@ -111,8 +120,12 @@ export async function predict(input: Tensor, config: Config): Promise { if (env.initial) model = null; - if (!model) model = await loadModel(config.face.mesh?.modelPath); - else if (config.debug) log('cached model:', model['modelUrl']); + if (!model) { + if (config.face.attention?.enabled) model = await loadModel(config.face.attention?.modelPath); + else model = await loadModel(config.face.mesh?.modelPath); + } else if (config.debug) { + log('cached model:', model['modelUrl']); + } inputSize = model.inputs[0].shape ? model.inputs[0].shape[2] : 0; return model; } diff --git a/src/face/iris.ts b/src/face/iris.ts index fde0a910..02ea2015 100644 --- a/src/face/iris.ts +++ b/src/face/iris.ts @@ -129,7 +129,7 @@ export async function augmentIris(rawCoords, face, config, meshSize) { const leftEyeData = eyePredictionsData.slice(0, irisLandmarks.numCoordinates * 3); const { rawCoords: leftEyeRawCoords, iris: leftIrisRawCoords } = getEyeCoords(leftEyeData, leftEyeBox, leftEyeBoxSize, true); const rightEyeData = eyePredictionsData.slice(irisLandmarks.numCoordinates * 3); - const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize); + const { rawCoords: rightEyeRawCoords, iris: rightIrisRawCoords } = getEyeCoords(rightEyeData, rightEyeBox, rightEyeBoxSize, false); const leftToRightEyeDepthDifference = getLeftToRightEyeDepthDifference(rawCoords); if (Math.abs(leftToRightEyeDepthDifference) < 30) { // User is looking straight ahead. replaceRawCoordinates(rawCoords, leftEyeRawCoords, 'left', null); diff --git a/src/human.ts b/src/human.ts index 0e7b938b..b0a4bc90 100644 --- a/src/human.ts +++ b/src/human.ts @@ -17,7 +17,7 @@ import * as app from '../package.json'; import * as backend from './tfjs/backend'; import * as blazepose from './body/blazepose'; import * as centernet from './object/centernet'; -import * as draw from './util/draw'; +import * as draw from './draw/draw'; import * as efficientpose from './body/efficientpose'; import * as face from './face/face'; import * as facemesh from './face/facemesh'; diff --git a/src/image/image.ts b/src/image/image.ts index ad1473e4..54e2409c 100644 --- a/src/image/image.ts +++ b/src/image/image.ts @@ -159,25 +159,29 @@ export async function process(input: Input, config: Config, getTensor: boolean = env.filter = !!fx; if (!fx || !fx.add) { if (config.debug) log('input process error: cannot initialize filters'); - return { tensor: null, canvas: inCanvas }; + env.webgl.supported = false; + config.filter.enabled = false; + copy(inCanvas, outCanvas); // filter failed to initialize + // return { tensor: null, canvas: inCanvas }; + } else { + fx.reset(); + if (config.filter.brightness !== 0) fx.add('brightness', config.filter.brightness); + if (config.filter.contrast !== 0) fx.add('contrast', config.filter.contrast); + if (config.filter.sharpness !== 0) fx.add('sharpen', config.filter.sharpness); + if (config.filter.blur !== 0) fx.add('blur', config.filter.blur); + if (config.filter.saturation !== 0) fx.add('saturation', config.filter.saturation); + if (config.filter.hue !== 0) fx.add('hue', config.filter.hue); + if (config.filter.negative) fx.add('negative'); + if (config.filter.sepia) fx.add('sepia'); + if (config.filter.vintage) fx.add('brownie'); + if (config.filter.sepia) fx.add('sepia'); + if (config.filter.kodachrome) fx.add('kodachrome'); + if (config.filter.technicolor) fx.add('technicolor'); + if (config.filter.polaroid) fx.add('polaroid'); + if (config.filter.pixelate !== 0) fx.add('pixelate', config.filter.pixelate); + if (fx.get() > 0) outCanvas = fx.apply(inCanvas); + else outCanvas = fx.draw(inCanvas); } - fx.reset(); - if (config.filter.brightness !== 0) fx.add('brightness', config.filter.brightness); - if (config.filter.contrast !== 0) fx.add('contrast', config.filter.contrast); - if (config.filter.sharpness !== 0) fx.add('sharpen', config.filter.sharpness); - if (config.filter.blur !== 0) fx.add('blur', config.filter.blur); - if (config.filter.saturation !== 0) fx.add('saturation', config.filter.saturation); - if (config.filter.hue !== 0) fx.add('hue', config.filter.hue); - if (config.filter.negative) fx.add('negative'); - if (config.filter.sepia) fx.add('sepia'); - if (config.filter.vintage) fx.add('brownie'); - if (config.filter.sepia) fx.add('sepia'); - if (config.filter.kodachrome) fx.add('kodachrome'); - if (config.filter.technicolor) fx.add('technicolor'); - if (config.filter.polaroid) fx.add('polaroid'); - if (config.filter.pixelate !== 0) fx.add('pixelate', config.filter.pixelate); - if (fx.get() > 0) outCanvas = fx.apply(inCanvas); - else outCanvas = fx.draw(inCanvas); } else { copy(inCanvas, outCanvas); // if no filters applied, output canvas is input canvas if (fx) fx = null; diff --git a/src/image/imagefx.ts b/src/image/imagefx.ts index dd57eefc..0db4ba2b 100644 --- a/src/image/imagefx.ts +++ b/src/image/imagefx.ts @@ -84,12 +84,12 @@ export function GLImageFilter() { const shaderProgramCache = { }; // key is the shader program source, value is the compiled program const DRAW = { INTERMEDIATE: 1 }; const gl = fxcanvas.getContext('webgl') as WebGLRenderingContext; - // @ts-ignore used for sanity checks outside of imagefx - this.gl = gl; if (!gl) { log('filter: cannot get webgl context'); return; } + // @ts-ignore used for sanity checks outside of imagefx + this.gl = gl; function resize(width, height) { if (width === fxcanvas.width && height === fxcanvas.height) return; // Same width/height? Nothing to do here diff --git a/test/build.log b/test/build.log index af65cb75..989ad2f8 100644 --- a/test/build.log +++ b/test/build.log @@ -1,24 +1,24 @@ -2022-04-10 10:11:37 INFO:  Application: {"name":"@vladmandic/human","version":"2.6.5"} -2022-04-10 10:11:37 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} -2022-04-10 10:11:37 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.34","typescript":"4.6.3","typedoc":"0.22.14","eslint":"8.13.0"} -2022-04-10 10:11:37 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} -2022-04-10 10:11:37 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":64,"inputBytes":564552,"outputBytes":293314} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":64,"inputBytes":564556,"outputBytes":293318} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":64,"inputBytes":564608,"outputBytes":293368} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1063,"outputBytes":394} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1068,"outputBytes":615} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":64,"inputBytes":564572,"outputBytes":292273} -2022-04-10 10:11:37 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1344714} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":64,"inputBytes":1908671,"outputBytes":1636003} -2022-04-10 10:11:37 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":64,"inputBytes":1908671,"outputBytes":2113825} -2022-04-10 10:11:43 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":112} -2022-04-10 10:11:45 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":72,"generated":true} -2022-04-10 10:11:45 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5862,"outputBytes":2915} -2022-04-10 10:11:45 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} -2022-04-10 10:11:54 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":96,"errors":0,"warnings":0} -2022-04-10 10:11:54 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} -2022-04-10 10:11:54 INFO:  Done... +2022-04-11 11:43:57 INFO:  Application: {"name":"@vladmandic/human","version":"2.7.0"} +2022-04-11 11:43:57 INFO:  Environment: {"profile":"production","config":".build.json","package":"package.json","tsconfig":true,"eslintrc":true,"git":true} +2022-04-11 11:43:57 INFO:  Toolchain: {"build":"0.7.2","esbuild":"0.14.34","typescript":"4.6.3","typedoc":"0.22.15","eslint":"8.13.0"} +2022-04-11 11:43:57 INFO:  Build: {"profile":"production","steps":["clean","compile","typings","typedoc","lint","changelog"]} +2022-04-11 11:43:57 STATE: Clean: {"locations":["dist/*","types/lib/*","typedoc/*"]} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/nodejs/cpu","format":"cjs","platform":"node","input":"tfjs/tf-node.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":102,"outputBytes":595} +2022-04-11 11:43:57 STATE: Compile: {"name":"human/nodejs/cpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node.js","files":72,"inputBytes":568981,"outputBytes":294190} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/nodejs/gpu","format":"cjs","platform":"node","input":"tfjs/tf-node-gpu.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":599} +2022-04-11 11:43:57 STATE: Compile: {"name":"human/nodejs/gpu","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-gpu.js","files":72,"inputBytes":568985,"outputBytes":294194} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/nodejs/wasm","format":"cjs","platform":"node","input":"tfjs/tf-node-wasm.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":149,"outputBytes":651} +2022-04-11 11:43:57 STATE: Compile: {"name":"human/nodejs/wasm","format":"cjs","platform":"node","input":"src/human.ts","output":"dist/human.node-wasm.js","files":72,"inputBytes":569037,"outputBytes":294244} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/browser/version","format":"esm","platform":"browser","input":"tfjs/tf-version.ts","output":"dist/tfjs.version.js","files":1,"inputBytes":1063,"outputBytes":394} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/browser/esm/nobundle","format":"esm","platform":"browser","input":"tfjs/tf-browser.ts","output":"dist/tfjs.esm.js","files":2,"inputBytes":1068,"outputBytes":615} +2022-04-11 11:43:57 STATE: Compile: {"name":"human/browser/esm/nobundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm-nobundle.js","files":72,"inputBytes":569001,"outputBytes":293151} +2022-04-11 11:43:57 STATE: Compile: {"name":"tfjs/browser/esm/custom","format":"esm","platform":"browser","input":"tfjs/tf-custom.ts","output":"dist/tfjs.esm.js","files":1,"inputBytes":110,"outputBytes":1344714} +2022-04-11 11:43:57 STATE: Compile: {"name":"human/browser/iife/bundle","format":"iife","platform":"browser","input":"src/human.ts","output":"dist/human.js","files":72,"inputBytes":1913100,"outputBytes":1636880} +2022-04-11 11:43:58 STATE: Compile: {"name":"human/browser/esm/bundle","format":"esm","platform":"browser","input":"src/human.ts","output":"dist/human.esm.js","files":72,"inputBytes":1913100,"outputBytes":2115805} +2022-04-11 11:44:03 STATE: Typings: {"input":"src/human.ts","output":"types/lib","files":114} +2022-04-11 11:44:05 STATE: TypeDoc: {"input":"src/human.ts","output":"typedoc","objects":73,"generated":true} +2022-04-11 11:44:05 STATE: Compile: {"name":"demo/typescript","format":"esm","platform":"browser","input":"demo/typescript/index.ts","output":"demo/typescript/index.js","files":1,"inputBytes":5916,"outputBytes":2957} +2022-04-11 11:44:05 STATE: Compile: {"name":"demo/faceid","format":"esm","platform":"browser","input":"demo/faceid/index.ts","output":"demo/faceid/index.js","files":2,"inputBytes":15174,"outputBytes":7820} +2022-04-11 11:44:13 STATE: Lint: {"locations":["*.json","src/**/*.ts","test/**/*.js","demo/**/*.js"],"files":104,"errors":0,"warnings":0} +2022-04-11 11:44:13 STATE: ChangeLog: {"repository":"https://github.com/vladmandic/human","branch":"main","output":"CHANGELOG.md"} +2022-04-11 11:44:13 INFO:  Done...