From 4cc5817ebfb64ccb546c05ef3abd8f95ac84a06e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 7 Jun 2021 08:38:16 -0400 Subject: [PATCH] add video drag&drop capability --- CHANGELOG.md | 4 +--- demo/index.js | 41 ++++++++++++++++++++++++++++++++--------- server/build.log | 44 ++++++++++++++++++++++---------------------- server/serve.js | 1 + 4 files changed, 56 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c8d1e19..3e2b0e3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,8 @@ Repository: **** ### **HEAD -> main** 2021/06/06 mandic00@live.com +- modularize build platform - custom build tfjs from sources - -### **update wasm to tfjs 3.7.0** 2021/06/06 mandic00@live.com - - modularize build platform - enable body segmentation and background replacement in demo - minor git corruption diff --git a/demo/index.js b/demo/index.js index 5ddd5d05..c771e2f4 100644 --- a/demo/index.js +++ b/demo/index.js @@ -432,17 +432,19 @@ function webWorker(input, image, canvas, timestamp) { // main processing function when input is webcam, can use direct invocation or web worker function runHumanDetect(input, canvas, timestamp) { // if live video - const live = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState > 2) && (!input.paused); - if (!live && input.srcObject) { + const videoLive = (input.readyState > 2) && (!input.paused); + const cameraLive = input.srcObject && (input.srcObject.getVideoTracks()[0].readyState === 'live'); + const live = videoLive || cameraLive; + if (!live) { // stop ui refresh if (ui.drawThread) cancelAnimationFrame(ui.drawThread); if (ui.detectThread) cancelAnimationFrame(ui.detectThread); ui.drawThread = null; ui.detectThread = null; // if we want to continue and camera not ready, retry in 0.5sec, else just give up - if (input.paused) log('camera paused'); - else if ((input.srcObject.getVideoTracks()[0].readyState === 'live') && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500); - else log(`camera not ready: track state: ${input.srcObject.getVideoTracks()[0].readyState} stream state: ${input.readyState}`); + if (input.paused) log('video paused'); + else if (cameraLive && (input.readyState <= 2)) setTimeout(() => runHumanDetect(input, canvas), 500); + else log(`video not ready: track state: ${input.srcObject ? input.srcObject.getVideoTracks()[0].readyState : 'unknown'} stream state: ${input.readyState}`); clearTimeout(ui.drawThread); ui.drawThread = null; log('frame statistics: process:', ui.framesDetect, 'refresh:', ui.framesDraw); @@ -552,12 +554,33 @@ async function processImage(input, title) { }); } +async function processVideo(input, title) { + status(`processing video: ${title}`); + const video = document.createElement('video'); + const canvas = document.getElementById('canvas'); + video.id = 'video-file'; + video.controls = true; + video.loop = true; + // video.onerror = async () => status(`video loading error: ${video.error.message}`); + video.addEventListener('error', () => status(`video loading error: ${video.error.message}`)); + video.addEventListener('canplay', async () => { + for (const m of Object.values(menu)) m.hide(); + document.getElementById('samples-container').style.display = 'none'; + document.getElementById('play').style.display = 'none'; + canvas.style.display = 'block'; + document.getElementById('btnStartText').innerHTML = 'pause video'; + await video.play(); + if (!ui.detectThread) runHumanDetect(video, canvas); + }); + video.src = input; +} + // just initialize everything and call main function async function detectVideo() { document.getElementById('samples-container').style.display = 'none'; - document.getElementById('canvas').style.display = 'block'; const video = document.getElementById('video'); const canvas = document.getElementById('canvas'); + canvas.style.display = 'block'; if ((video.srcObject !== null) && !video.paused) { document.getElementById('btnStartText').innerHTML = 'start video'; status('paused'); @@ -743,9 +766,9 @@ async function processDataURL(f, action) { return new Promise((resolve) => { const reader = new FileReader(); reader.onload = async (e) => { - const dataURL = e.target.result; if (action === 'process') { - await processImage(dataURL, f.name); + if (e.target.result.startsWith('data:image')) await processImage(e.target.result, f.name); + if (e.target.result.startsWith('data:video')) await processVideo(e.target.result, f.name); document.getElementById('canvas').style.display = 'none'; } if (action === 'background') { @@ -767,7 +790,7 @@ async function processDataURL(f, action) { } } }; - image.src = dataURL; + image.src = e.target.result; } resolve(true); }; diff --git a/server/build.log b/server/build.log index e7db621f..e524ae11 100644 --- a/server/build.log +++ b/server/build.log @@ -1,22 +1,22 @@ -2021-06-06 20:46:24 INFO:  @vladmandic/human version 2.0.0 -2021-06-06 20:46:24 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 -2021-06-06 20:46:24 INFO:  Toolchain: tfjs: 3.7.0 esbuild 0.12.6; typescript 4.2.4; typedoc: 0.20.36 eslint: 7.28.0 -2021-06-06 20:46:24 INFO:  Clean: ["dist/*","types/*","typedoc/*"] -2021-06-06 20:46:24 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} -2021-06-06 20:46:24 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"} -2021-06-06 20:46:24 STATE: target: node type: node: {"imports":41,"importBytes":430471,"outputBytes":376423,"outputFiles":"dist/human.node.js"} -2021-06-06 20:46:24 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"} -2021-06-06 20:46:24 STATE: target: nodeGPU type: node: {"imports":41,"importBytes":430479,"outputBytes":376427,"outputFiles":"dist/human.node-gpu.js"} -2021-06-06 20:46:24 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"} -2021-06-06 20:46:24 STATE: target: nodeWASM type: node: {"imports":41,"importBytes":430546,"outputBytes":376499,"outputFiles":"dist/human.node-wasm.js"} -2021-06-06 20:46:24 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2817,"outputBytes":1214,"outputFiles":"dist/tfjs.esm.js"} -2021-06-06 20:46:24 STATE: target: browserNoBundle type: esm: {"imports":41,"importBytes":430382,"outputBytes":247743,"outputFiles":"dist/human.esm-nobundle.js"} -2021-06-06 20:46:25 STATE: target: browserBundle type: tfjs: {"modules":1684,"moduleBytes":5720339,"imports":7,"importBytes":2817,"outputBytes":2817757,"outputFiles":"dist/tfjs.esm.js"} -2021-06-06 20:46:25 STATE: target: browserBundle type: iife: {"imports":41,"importBytes":3246925,"outputBytes":1588120,"outputFiles":"dist/human.js"} -2021-06-06 20:46:26 STATE: target: browserBundle type: esm: {"imports":41,"importBytes":3246925,"outputBytes":1588112,"outputFiles":"dist/human.esm.js"} -2021-06-06 20:46:26 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"] -2021-06-06 20:46:50 INFO:  Linter complete: files: 71 errors: 0 warnings: 0 -2021-06-06 20:46:50 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"] -2021-06-06 20:46:50 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"] -2021-06-06 20:47:05 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"] -2021-06-06 20:47:17 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 +2021-06-07 08:36:47 INFO:  @vladmandic/human version 2.0.0 +2021-06-07 08:36:47 INFO:  User: vlado Platform: linux Arch: x64 Node: v16.0.0 +2021-06-07 08:36:47 INFO:  Toolchain: tfjs: 3.7.0 esbuild 0.12.6; typescript 4.2.4; typedoc: 0.20.36 eslint: 7.28.0 +2021-06-07 08:36:47 INFO:  Clean: ["dist/*","types/*","typedoc/*"] +2021-06-07 08:36:47 INFO:  Build: file startup all type: production config: {"minifyWhitespace":true,"minifyIdentifiers":true,"minifySyntax":true} +2021-06-07 08:36:47 STATE: target: node type: tfjs: {"imports":1,"importBytes":102,"outputBytes":1303,"outputFiles":"dist/tfjs.esm.js"} +2021-06-07 08:36:47 STATE: target: node type: node: {"imports":41,"importBytes":430471,"outputBytes":376423,"outputFiles":"dist/human.node.js"} +2021-06-07 08:36:47 STATE: target: nodeGPU type: tfjs: {"imports":1,"importBytes":110,"outputBytes":1311,"outputFiles":"dist/tfjs.esm.js"} +2021-06-07 08:36:47 STATE: target: nodeGPU type: node: {"imports":41,"importBytes":430479,"outputBytes":376427,"outputFiles":"dist/human.node-gpu.js"} +2021-06-07 08:36:47 STATE: target: nodeWASM type: tfjs: {"imports":1,"importBytes":149,"outputBytes":1378,"outputFiles":"dist/tfjs.esm.js"} +2021-06-07 08:36:47 STATE: target: nodeWASM type: node: {"imports":41,"importBytes":430546,"outputBytes":376499,"outputFiles":"dist/human.node-wasm.js"} +2021-06-07 08:36:47 STATE: target: browserNoBundle type: tfjs: {"imports":1,"importBytes":2817,"outputBytes":1214,"outputFiles":"dist/tfjs.esm.js"} +2021-06-07 08:36:47 STATE: target: browserNoBundle type: esm: {"imports":41,"importBytes":430382,"outputBytes":247743,"outputFiles":"dist/human.esm-nobundle.js"} +2021-06-07 08:36:48 STATE: target: browserBundle type: tfjs: {"modules":1684,"moduleBytes":5720339,"imports":7,"importBytes":2817,"outputBytes":2817757,"outputFiles":"dist/tfjs.esm.js"} +2021-06-07 08:36:48 STATE: target: browserBundle type: iife: {"imports":41,"importBytes":3246925,"outputBytes":1588120,"outputFiles":"dist/human.js"} +2021-06-07 08:36:49 STATE: target: browserBundle type: esm: {"imports":41,"importBytes":3246925,"outputBytes":1588112,"outputFiles":"dist/human.esm.js"} +2021-06-07 08:36:49 INFO:  Running Linter: ["server/","src/","tfjs/","test/","demo/"] +2021-06-07 08:37:13 INFO:  Linter complete: files: 71 errors: 0 warnings: 0 +2021-06-07 08:37:13 INFO:  Generate ChangeLog: ["/home/vlado/dev/human/CHANGELOG.md"] +2021-06-07 08:37:13 INFO:  Generate Typings: ["src/human.ts"] outDir: ["types"] +2021-06-07 08:37:28 INFO:  Generate TypeDocs: ["src/human.ts"] outDir: ["typedoc"] +2021-06-07 08:37:41 INFO:  Documentation generated at /home/vlado/dev/human/typedoc 1 diff --git a/server/serve.js b/server/serve.js index 711c974d..46769b5b 100644 --- a/server/serve.js +++ b/server/serve.js @@ -143,6 +143,7 @@ async function httpRequest(req, res) { 'X-Content-Type-Options': 'nosniff', 'Cross-Origin-Embedder-Policy': 'require-corp', 'Cross-Origin-Opener-Policy': 'same-origin', + 'Content-Security-Policy': "media-src 'self' http: https: data:", }); const compress = zlib.createBrotliCompress({ params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 5 } }); // instance of brotli compression with level 5 const stream = fs.createReadStream(input);