From 468898690146164334f69563b747986ce759de5d Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Sun, 9 Oct 2022 04:07:45 +0200 Subject: [PATCH 1/4] Resolve favicon locally (WIP) --- components/common/Favicon.js | 21 ++------ components/metrics/WebsiteChart.js | 4 +- components/metrics/WebsiteHeader.js | 6 +-- components/pages/TestConsole.js | 2 +- components/pages/WebsiteDetails.js | 2 +- components/pages/WebsiteList.js | 4 +- components/settings/WebsiteSettings.js | 4 +- .../04_add_favicon_column/migration.sql | 2 + db/postgresql/schema.prisma | 1 + package.json | 1 + pages/api/website/index.js | 28 ++++++++++- public/default-favicon.png | Bin 0 -> 1478 bytes yarn.lock | 46 ++++++++++++++++++ 13 files changed, 91 insertions(+), 30 deletions(-) create mode 100644 db/postgresql/migrations/04_add_favicon_column/migration.sql create mode 100644 public/default-favicon.png diff --git a/components/common/Favicon.js b/components/common/Favicon.js index d72cd3c7..00376b05 100644 --- a/components/common/Favicon.js +++ b/components/common/Favicon.js @@ -2,27 +2,14 @@ import React from 'react'; import PropTypes from 'prop-types'; import styles from './Favicon.module.css'; -function getHostName(url) { - const match = url.match(/^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:/\n?=]+)/im); - return match && match.length > 1 ? match[1] : null; -} +function Favicon({ url, ...props }) { + const faviconUrl = url ? url : '/default-favicon.png'; -function Favicon({ domain, ...props }) { - const hostName = domain ? getHostName(domain) : null; - - return hostName ? ( - - ) : null; + return ; } Favicon.propTypes = { - domain: PropTypes.string, + url: PropTypes.string, }; export default Favicon; diff --git a/components/metrics/WebsiteChart.js b/components/metrics/WebsiteChart.js index fd74538f..5e3c09e6 100644 --- a/components/metrics/WebsiteChart.js +++ b/components/metrics/WebsiteChart.js @@ -18,7 +18,7 @@ import styles from './WebsiteChart.module.css'; export default function WebsiteChart({ websiteId, title, - domain, + favicon, stickyHeader = false, showLink = false, showChart = true, @@ -81,7 +81,7 @@ export default function WebsiteChart({ return (
- +
- + ) : ( <> - + {title} ); diff --git a/components/pages/TestConsole.js b/components/pages/TestConsole.js index efeb0264..1f36c0b5 100644 --- a/components/pages/TestConsole.js +++ b/components/pages/TestConsole.js @@ -106,7 +106,7 @@ export default function TestConsole() { Events diff --git a/components/pages/WebsiteDetails.js b/components/pages/WebsiteDetails.js index 78c5f752..a35d02a0 100644 --- a/components/pages/WebsiteDetails.js +++ b/components/pages/WebsiteDetails.js @@ -141,7 +141,7 @@ export default function WebsiteDetails({ websiteId }) { - {ordered.map(({ website_id, name, domain }, index) => + {ordered.map(({ website_id, name, favicon }, index) => index < limit ? (
diff --git a/components/settings/WebsiteSettings.js b/components/settings/WebsiteSettings.js index 451be47f..22dfc1ad 100644 --- a/components/settings/WebsiteSettings.js +++ b/components/settings/WebsiteSettings.js @@ -83,13 +83,13 @@ export default function WebsiteSettings() { ); - const DetailsLink = ({ website_id, name, domain }) => ( + const DetailsLink = ({ website_id, name, favicon }) => ( - + {name} ); diff --git a/db/postgresql/migrations/04_add_favicon_column/migration.sql b/db/postgresql/migrations/04_add_favicon_column/migration.sql new file mode 100644 index 00000000..a1f113ca --- /dev/null +++ b/db/postgresql/migrations/04_add_favicon_column/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "website" ADD COLUMN "favicon" VARCHAR(500); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index a76a3da4..7c661d2b 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -85,6 +85,7 @@ model website { domain String? @db.VarChar(500) share_id String? @unique @db.VarChar(64) created_at DateTime? @default(now()) @db.Timestamptz(6) + favicon String? @db.VarChar(500) account account @relation(fields: [user_id], references: [user_id]) event event[] pageview pageview[] diff --git a/package.json b/package.json index be7fd7d1..0019e2e0 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "del": "^6.0.0", "detect-browser": "^5.2.0", "dotenv": "^10.0.0", + "favecon": "^1.0.2", "formik": "^2.2.9", "fs-extra": "^10.0.1", "immer": "^9.0.12", diff --git a/pages/api/website/index.js b/pages/api/website/index.js index ac02de85..9f57b9a8 100644 --- a/pages/api/website/index.js +++ b/pages/api/website/index.js @@ -2,6 +2,22 @@ import { ok, unauthorized, methodNotAllowed, getRandomChars } from 'next-basics' import { updateWebsite, createWebsite, getWebsiteById } from 'queries'; import { useAuth } from 'lib/middleware'; import { uuid } from 'lib/crypto'; +import favecon from 'favecon'; + +const getFavicon = async domain => { + try { + const icons = await favecon.getIcons(`https://${domain}`); + + if (icons.length && icons.length > 0) { + return icons[0]?.href; + } + } catch (e) { + // eslint-disable-next-line no-console + console.log(`Could not fetch favicon for domain ${domain}`, e); + } + + return null; +}; export default async (req, res) => { await useAuth(req, res); @@ -13,6 +29,8 @@ export default async (req, res) => { const { name, domain, owner } = req.body; const website_owner = parseInt(owner); + const favicon = await getFavicon(domain); + if (website_id) { const website = await getWebsiteById(website_id); @@ -28,13 +46,19 @@ export default async (req, res) => { share_id = null; } - await updateWebsite(website_id, { name, domain, share_id, user_id: website_owner }); + await updateWebsite(website_id, { name, domain, share_id, user_id: website_owner, favicon }); return ok(res); } else { const website_uuid = uuid(); const share_id = enable_share_url ? getRandomChars(8) : null; - const website = await createWebsite(website_owner, { website_uuid, name, domain, share_id }); + const website = await createWebsite(website_owner, { + website_uuid, + name, + domain, + share_id, + favicon, + }); return ok(res, website); } diff --git a/public/default-favicon.png b/public/default-favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..b018b02cce8f6f854ecf9721e172411914a05f8c GIT binary patch literal 1478 zcmeAS@N?(olHy`uVBq!ia0vp^1|ZDA1|-9oezpTC$r9IylHmNblJdl&R0hYC{G?O` z&)mfH)S%SFl*+=BsWuD@%o>>?5hW46K32*3xq68pHF_1f1wh>l3^w)^1&PVosU-?Y zsp*+{wo31J?^jaDOtDo8H}y5}EpSfF$n>ZxN)4{^3rViZPPR-@vbR&PsjvbXkegbP zs8ErclUHn2VXFi-*9yo63F|8hm3bwJ6}oxF$}kgLQj3#|G7CyF^YauyCMG83 zmzLNn0bL65LT&-v*t}wBFaZNhzap_f-%!s0*lEl2^R8JRMC7?NanVBh8PEHngVhK8;tPHu)4PHs*X zrUn+KmPTf7FugAM$)&lec_lEtDG0q5IQ4=OL~a4lW|!2W%(B!Jx1#)91+d4hGI6`b z6sLJmy(zfeVun+%KF~4xpy)-4ZkP}-4S|^O#0%uWlYeR+FwGYM6Sv{2XCVv>OeUT# zjv*DduFSISJ7gfxx>roLxQj5;vQJ9{ojRV>eoL|O{8MT-pS3YbR{Kwn!K#pi6sw{?GE0woT#lCYI=<;~J=s zS4L;sOkc#*5NQAHM6u(PD0iFh8V${Tb2*mFR!OiF9O%~GV&*HbNila>!o9W|<*WgX zmzQbI4Uu-ZGOf6=v>?1Pa-!8~PG+6EA(PH=d51oaw|dv}+9hVIkb29r#3}c#v3jgM zvPI_7OSP3}=g&wxRI*RLLASZ`AkW6Fj3R;uc0V)c<49#lT6Ai2rslyb0uL;wx$KnZ zN>Y70$;7$izhp;^c|wbTpm>9RhX+^OM&|=hdg=7ySO&m4EoY%FSf=55Tpfc+q2eMwEV(f^M*40$dLnXD|=h4;$3mZVT6V z(_&Xj Date: Sun, 9 Oct 2022 04:48:00 +0200 Subject: [PATCH 2/4] Use own favicon fetcher --- lib/favicon.js | 57 +++++++++++++++++++++ package.json | 3 +- pages/api/website/index.js | 17 +------ yarn.lock | 100 ++++++++++++++++++++++--------------- 4 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 lib/favicon.js diff --git a/lib/favicon.js b/lib/favicon.js new file mode 100644 index 00000000..d2ef8cb3 --- /dev/null +++ b/lib/favicon.js @@ -0,0 +1,57 @@ +import fetch from 'node-fetch'; +import AbortController from 'abort-controller'; +import { parse } from 'node-html-parser'; + +//Node.js >=14.17 only +//const AbortController = globalThis.AbortController || (await import('abort-controller')); + +const filterLinks = links => { + const attrs = ['rel', 'href', 'sizes']; + const filterAttrs = link => + attrs.reduce((total, attr) => ({ ...total, [attr]: link.getAttribute(attr) }), {}); + return [...links] + .filter(link => link.getAttribute('href') && link.getAttribute('rel').includes('icon')) + .map(filterAttrs); +}; + +const updateAttrs = url => icons => { + const getOrigin = url => new URL(url).origin; + return icons.map(({ sizes, href, rel }) => ({ + size: parseInt(sizes?.split('x')[0]) || undefined, + href: href[0] === '/' ? `${getOrigin(url)}${href}` : href, + rel, + })); +}; + +const fetchLinks = url => { + const controller = new AbortController(); + const timeout = setTimeout(() => { + // eslint-disable-next-line no-console + console.log('Aborting!'); + controller.abort(); + }, 1000); + + try { + return fetch(url, { signal: controller.signal }) + .then(res => res.text()) + .then(str => parse(str)) + .then(html => html.querySelectorAll('head link')); + } catch (e) { + // eslint-disable-next-line no-console + console.log(`Could not fetch favicon for url ${url}`, e); + } finally { + clearTimeout(timeout); + } +}; + +const getIcons = url => fetchLinks(url).then(filterLinks).then(updateAttrs(url)); + +export default async domain => { + const icons = await getIcons(`https://${domain}`); + + if (icons.length && icons.length > 0) { + return icons[0]?.href; + } + + return null; +}; diff --git a/package.json b/package.json index 0019e2e0..e48baebe 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "dependencies": { "@fontsource/inter": "4.5.7", "@prisma/client": "4.3.1", + "abort-controller": "^3.0.0", "chalk": "^4.1.1", "chart.js": "^2.9.4", "classnames": "^2.3.1", @@ -71,7 +72,6 @@ "del": "^6.0.0", "detect-browser": "^5.2.0", "dotenv": "^10.0.0", - "favecon": "^1.0.2", "formik": "^2.2.9", "fs-extra": "^10.0.1", "immer": "^9.0.12", @@ -87,6 +87,7 @@ "next": "^12.2.5", "next-basics": "^0.7.0", "node-fetch": "^3.2.8", + "node-html-parser": "^6.1.1", "npm-run-all": "^4.1.5", "prop-types": "^15.7.2", "react": "^17.0.0", diff --git a/pages/api/website/index.js b/pages/api/website/index.js index 9f57b9a8..527a6a43 100644 --- a/pages/api/website/index.js +++ b/pages/api/website/index.js @@ -2,22 +2,7 @@ import { ok, unauthorized, methodNotAllowed, getRandomChars } from 'next-basics' import { updateWebsite, createWebsite, getWebsiteById } from 'queries'; import { useAuth } from 'lib/middleware'; import { uuid } from 'lib/crypto'; -import favecon from 'favecon'; - -const getFavicon = async domain => { - try { - const icons = await favecon.getIcons(`https://${domain}`); - - if (icons.length && icons.length > 0) { - return icons[0]?.href; - } - } catch (e) { - // eslint-disable-next-line no-console - console.log(`Could not fetch favicon for domain ${domain}`, e); - } - - return null; -}; +import getFavicon from 'lib/favicon'; export default async (req, res) => { await useAuth(req, res); diff --git a/yarn.lock b/yarn.lock index 19728cf0..0ecada3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1978,6 +1978,13 @@ JSONStream@1.3.4: jsonparse "^1.2.0" through ">=2.2.7 <3" +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + acorn-dynamic-import@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz" @@ -2674,6 +2681,17 @@ css-select@^4.1.3: domutils "^2.8.0" nth-check "^2.0.1" +css-select@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6" + integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + css-tree@^1.1.2, css-tree@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" @@ -2682,7 +2700,7 @@ css-tree@^1.1.2, css-tree@^1.1.3: mdn-data "2.0.14" source-map "^0.6.1" -css-what@^6.0.1: +css-what@^6.0.1, css-what@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== @@ -2962,7 +2980,16 @@ dom-serializer@^1.0.1: domhandler "^4.2.0" entities "^2.0.0" -domelementtype@^2.0.1, domelementtype@^2.2.0: +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== @@ -2974,6 +3001,13 @@ domhandler@^4.2.0, domhandler@^4.3.1: dependencies: domelementtype "^2.2.0" +domhandler@^5.0.1, domhandler@^5.0.2: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + domutils@^2.8.0: version "2.8.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" @@ -2983,6 +3017,15 @@ domutils@^2.8.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.0.1.tgz#696b3875238338cb186b6c0612bd4901c89a4f1c" + integrity sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.1" + dotenv@^10.0.0: version "10.0.0" resolved "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz" @@ -3035,6 +3078,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^4.2.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174" + integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA== + entities@^4.3.0: version "4.3.1" resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.1.tgz#c34062a94c865c322f9d67b4384e4169bcede6a4" @@ -3357,6 +3405,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + execa@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz" @@ -3451,14 +3504,6 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" -favecon@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/favecon/-/favecon-1.0.2.tgz#450ae94066900be474b9e408799ecf6e4a65658c" - integrity sha512-2jpgF7sMheI/UBGJLzi4JyknC26Bly1txL3rCNNxsY9yHp4lflB3kEYX79zZ/JeSqmQAf8tbktqonVkuj+iwOQ== - dependencies: - node-fetch "^2.6.1" - node-html-parser "^4.0.0" - fetch-blob@^3.1.2, fetch-blob@^3.1.4: version "3.2.0" resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz" @@ -4788,13 +4833,6 @@ node-domexception@^1.0.0: resolved "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@^2.6.1: - version "2.6.7" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" - integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== - dependencies: - whatwg-url "^5.0.0" - node-fetch@^3.2.8: version "3.2.10" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-3.2.10.tgz" @@ -4804,12 +4842,12 @@ node-fetch@^3.2.8: fetch-blob "^3.1.4" formdata-polyfill "^4.0.10" -node-html-parser@^4.0.0: - version "4.1.5" - resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-4.1.5.tgz#e3ff5b39a098e70de3629c9c79c4e29a3fa5f062" - integrity sha512-NLgqUXtftqnBqIjlRjYSaApaqE7TTxfTiH4VqKCjdUJKFOtUzRwney83EHz2qYc0XoxXAkYdmLjENCuZHvsIFg== +node-html-parser@^6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-6.1.1.tgz#7f38f4427fafc242a22135d9db80c1455e837467" + integrity sha512-eYYblUeoMg0nR6cYGM4GRb1XncNa9FXEftuKAU1qyMIr6rXVtNyUKduvzZtkqFqSHVByq2lLjC7WO8tz7VDmnA== dependencies: - css-select "^4.1.3" + css-select "^5.1.0" he "1.2.0" node-releases@^2.0.3: @@ -6503,11 +6541,6 @@ tough-cookie@~2.4.3: psl "^1.1.24" punycode "^1.4.1" -tr46@~0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== - trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" @@ -6760,19 +6793,6 @@ web-streams-polyfill@^3.0.3: resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz" integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== -webidl-conversions@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== - -whatwg-url@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== - dependencies: - tr46 "~0.0.3" - webidl-conversions "^3.0.0" - which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" From 76f951093f9f0eac71779b8263aca77731ea95e6 Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Sun, 9 Oct 2022 18:23:30 +0200 Subject: [PATCH 3/4] Timeout fixes and cleanup --- .../04_add_favicon_column/migration.sql | 2 ++ db/mysql/schema.prisma | 1 + lib/favicon.js | 35 +++++++++---------- package.json | 1 + yarn.lock | 12 +++++++ 5 files changed, 33 insertions(+), 18 deletions(-) create mode 100644 db/mysql/migrations/04_add_favicon_column/migration.sql diff --git a/db/mysql/migrations/04_add_favicon_column/migration.sql b/db/mysql/migrations/04_add_favicon_column/migration.sql new file mode 100644 index 00000000..f344c8ff --- /dev/null +++ b/db/mysql/migrations/04_add_favicon_column/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE `website` ADD COLUMN `favicon` VARCHAR(500) NULL; diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 9ad2620c..0daba55a 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -85,6 +85,7 @@ model website { domain String? @db.VarChar(500) share_id String? @unique() @db.VarChar(64) created_at DateTime? @default(now()) @db.Timestamp(0) + favicon String? @db.VarChar(500) account account @relation(fields: [user_id], references: [user_id]) event event[] pageview pageview[] diff --git a/lib/favicon.js b/lib/favicon.js index d2ef8cb3..c39df37b 100644 --- a/lib/favicon.js +++ b/lib/favicon.js @@ -1,9 +1,6 @@ import fetch from 'node-fetch'; -import AbortController from 'abort-controller'; import { parse } from 'node-html-parser'; - -//Node.js >=14.17 only -//const AbortController = globalThis.AbortController || (await import('abort-controller')); +import { TimeoutController } from 'timeout-abort-controller'; const filterLinks = links => { const attrs = ['rel', 'href', 'sizes']; @@ -14,7 +11,7 @@ const filterLinks = links => { .map(filterAttrs); }; -const updateAttrs = url => icons => { +const formatValues = (url, icons) => { const getOrigin = url => new URL(url).origin; return icons.map(({ sizes, href, rel }) => ({ size: parseInt(sizes?.split('x')[0]) || undefined, @@ -23,28 +20,30 @@ const updateAttrs = url => icons => { })); }; -const fetchLinks = url => { - const controller = new AbortController(); - const timeout = setTimeout(() => { - // eslint-disable-next-line no-console - console.log('Aborting!'); - controller.abort(); - }, 1000); +const fetchLinks = async url => { + // Time out the favicon request if it doesn't respond in a reasonable time. + const tc = new TimeoutController(3000); + + let links = []; try { - return fetch(url, { signal: controller.signal }) - .then(res => res.text()) - .then(str => parse(str)) - .then(html => html.querySelectorAll('head link')); + const html = await (await fetch(url, { signal: tc.signal })).text(); + links = parse(html).querySelectorAll('head link'); } catch (e) { // eslint-disable-next-line no-console console.log(`Could not fetch favicon for url ${url}`, e); } finally { - clearTimeout(timeout); + tc.clear(); } + + return links; }; -const getIcons = url => fetchLinks(url).then(filterLinks).then(updateAttrs(url)); +const getIcons = async url => { + const links = await fetchLinks(url); + const icons = filterLinks(links); + return formatValues(url, icons); +}; export default async domain => { const icons = await getIcons(`https://${domain}`); diff --git a/package.json b/package.json index e48baebe..099e26d6 100644 --- a/package.json +++ b/package.json @@ -102,6 +102,7 @@ "request-ip": "^3.3.0", "semver": "^7.3.6", "thenby": "^1.3.4", + "timeout-abort-controller": "^3.0.0", "timezone-support": "^2.0.2", "uuid": "^8.3.2", "zustand": "^3.7.2" diff --git a/yarn.lock b/yarn.lock index 0ecada3e..5d9a5203 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5904,6 +5904,11 @@ restore-cursor@^3.1.0: onetime "^5.1.0" signal-exit "^3.0.2" +retimer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/retimer/-/retimer-3.0.0.tgz#98b751b1feaf1af13eb0228f8ea68b8f9da530df" + integrity sha512-WKE0j11Pa0ZJI5YIk0nflGI7SQsfl2ljihVy7ogh7DeQSeYAUi0ubZ/yEueGtDfUPk6GH5LRw1hBdLq4IwUBWA== + reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" @@ -6492,6 +6497,13 @@ through@2.3.8, "through@>=2.2.7 <3", through@^2.3.8: resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +timeout-abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/timeout-abort-controller/-/timeout-abort-controller-3.0.0.tgz#dd57ffca041652c03769904f8d95afd93fb95595" + integrity sha512-O3e+2B8BKrQxU2YRyEjC/2yFdb33slI22WRdUaDx6rvysfi9anloNZyR2q0l6LnePo5qH7gSM7uZtvvwZbc2yA== + dependencies: + retimer "^3.0.0" + timezone-support@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/timezone-support/-/timezone-support-2.0.2.tgz" From 374f71c85928421306b7f8a6d4d33f7f49db6fda Mon Sep 17 00:00:00 2001 From: Stanislav Khromov Date: Sun, 9 Oct 2022 18:43:17 +0200 Subject: [PATCH 4/4] Add license --- lib/favicon.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/favicon.js b/lib/favicon.js index c39df37b..9110aa58 100644 --- a/lib/favicon.js +++ b/lib/favicon.js @@ -1,3 +1,11 @@ +/** + * ISC License + * Copyright 2022 github.com/sudoaugustin/favecon + * Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN + * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ import fetch from 'node-fetch'; import { parse } from 'node-html-parser'; import { TimeoutController } from 'timeout-abort-controller';