Use own favicon fetcher
Stanislav Khromov
parent
4688986901
commit
88b52d23cd
|
|
@ -0,0 +1,57 @@
|
|||
import fetch from 'node-fetch';
|
||||
import AbortController from 'abort-controller';
|
||||
import { parse } from 'node-html-parser';
|
||||
|
||||
//Node.js >=14.17 only
|
||||
//const AbortController = globalThis.AbortController || (await import('abort-controller'));
|
||||
|
||||
const filterLinks = links => {
|
||||
const attrs = ['rel', 'href', 'sizes'];
|
||||
const filterAttrs = link =>
|
||||
attrs.reduce((total, attr) => ({ ...total, [attr]: link.getAttribute(attr) }), {});
|
||||
return [...links]
|
||||
.filter(link => link.getAttribute('href') && link.getAttribute('rel').includes('icon'))
|
||||
.map(filterAttrs);
|
||||
};
|
||||
|
||||
const updateAttrs = url => icons => {
|
||||
const getOrigin = url => new URL(url).origin;
|
||||
return icons.map(({ sizes, href, rel }) => ({
|
||||
size: parseInt(sizes?.split('x')[0]) || undefined,
|
||||
href: href[0] === '/' ? `${getOrigin(url)}${href}` : href,
|
||||
rel,
|
||||
}));
|
||||
};
|
||||
|
||||
const fetchLinks = url => {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Aborting!');
|
||||
controller.abort();
|
||||
}, 1000);
|
||||
|
||||
try {
|
||||
return fetch(url, { signal: controller.signal })
|
||||
.then(res => res.text())
|
||||
.then(str => parse(str))
|
||||
.then(html => html.querySelectorAll('head link'));
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Could not fetch favicon for url ${url}`, e);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
};
|
||||
|
||||
const getIcons = url => fetchLinks(url).then(filterLinks).then(updateAttrs(url));
|
||||
|
||||
export default async domain => {
|
||||
const icons = await getIcons(`https://${domain}`);
|
||||
|
||||
if (icons.length && icons.length > 0) {
|
||||
return icons[0]?.href;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
|
@ -57,6 +57,7 @@
|
|||
"dependencies": {
|
||||
"@fontsource/inter": "4.5.7",
|
||||
"@prisma/client": "4.3.1",
|
||||
"abort-controller": "^3.0.0",
|
||||
"chalk": "^4.1.1",
|
||||
"chart.js": "^2.9.4",
|
||||
"classnames": "^2.3.1",
|
||||
|
|
@ -71,7 +72,6 @@
|
|||
"del": "^6.0.0",
|
||||
"detect-browser": "^5.2.0",
|
||||
"dotenv": "^10.0.0",
|
||||
"favecon": "^1.0.2",
|
||||
"formik": "^2.2.9",
|
||||
"fs-extra": "^10.0.1",
|
||||
"immer": "^9.0.12",
|
||||
|
|
@ -87,6 +87,7 @@
|
|||
"next": "^12.2.5",
|
||||
"next-basics": "^0.7.0",
|
||||
"node-fetch": "^3.2.8",
|
||||
"node-html-parser": "^6.1.1",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"prop-types": "^15.7.2",
|
||||
"react": "^17.0.0",
|
||||
|
|
|
|||
|
|
@ -2,22 +2,7 @@ import { ok, unauthorized, methodNotAllowed, getRandomChars } from 'next-basics'
|
|||
import { updateWebsite, createWebsite, getWebsiteById } from 'queries';
|
||||
import { useAuth } from 'lib/middleware';
|
||||
import { uuid } from 'lib/crypto';
|
||||
import favecon from 'favecon';
|
||||
|
||||
const getFavicon = async domain => {
|
||||
try {
|
||||
const icons = await favecon.getIcons(`https://${domain}`);
|
||||
|
||||
if (icons.length && icons.length > 0) {
|
||||
return icons[0]?.href;
|
||||
}
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`Could not fetch favicon for domain ${domain}`, e);
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
import getFavicon from 'lib/favicon';
|
||||
|
||||
export default async (req, res) => {
|
||||
await useAuth(req, res);
|
||||
|
|
|
|||
100
yarn.lock
100
yarn.lock
|
|
@ -1978,6 +1978,13 @@ JSONStream@1.3.4:
|
|||
jsonparse "^1.2.0"
|
||||
through ">=2.2.7 <3"
|
||||
|
||||
abort-controller@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
|
||||
integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
|
||||
dependencies:
|
||||
event-target-shim "^5.0.0"
|
||||
|
||||
acorn-dynamic-import@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-4.0.0.tgz"
|
||||
|
|
@ -2674,6 +2681,17 @@ css-select@^4.1.3:
|
|||
domutils "^2.8.0"
|
||||
nth-check "^2.0.1"
|
||||
|
||||
css-select@^5.1.0:
|
||||
version "5.1.0"
|
||||
resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.1.0.tgz#b8ebd6554c3637ccc76688804ad3f6a6fdaea8a6"
|
||||
integrity sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==
|
||||
dependencies:
|
||||
boolbase "^1.0.0"
|
||||
css-what "^6.1.0"
|
||||
domhandler "^5.0.2"
|
||||
domutils "^3.0.1"
|
||||
nth-check "^2.0.1"
|
||||
|
||||
css-tree@^1.1.2, css-tree@^1.1.3:
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d"
|
||||
|
|
@ -2682,7 +2700,7 @@ css-tree@^1.1.2, css-tree@^1.1.3:
|
|||
mdn-data "2.0.14"
|
||||
source-map "^0.6.1"
|
||||
|
||||
css-what@^6.0.1:
|
||||
css-what@^6.0.1, css-what@^6.1.0:
|
||||
version "6.1.0"
|
||||
resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4"
|
||||
integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==
|
||||
|
|
@ -2962,7 +2980,16 @@ dom-serializer@^1.0.1:
|
|||
domhandler "^4.2.0"
|
||||
entities "^2.0.0"
|
||||
|
||||
domelementtype@^2.0.1, domelementtype@^2.2.0:
|
||||
dom-serializer@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53"
|
||||
integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==
|
||||
dependencies:
|
||||
domelementtype "^2.3.0"
|
||||
domhandler "^5.0.2"
|
||||
entities "^4.2.0"
|
||||
|
||||
domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0:
|
||||
version "2.3.0"
|
||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d"
|
||||
integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==
|
||||
|
|
@ -2974,6 +3001,13 @@ domhandler@^4.2.0, domhandler@^4.3.1:
|
|||
dependencies:
|
||||
domelementtype "^2.2.0"
|
||||
|
||||
domhandler@^5.0.1, domhandler@^5.0.2:
|
||||
version "5.0.3"
|
||||
resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31"
|
||||
integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==
|
||||
dependencies:
|
||||
domelementtype "^2.3.0"
|
||||
|
||||
domutils@^2.8.0:
|
||||
version "2.8.0"
|
||||
resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
|
||||
|
|
@ -2983,6 +3017,15 @@ domutils@^2.8.0:
|
|||
domelementtype "^2.2.0"
|
||||
domhandler "^4.2.0"
|
||||
|
||||
domutils@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.0.1.tgz#696b3875238338cb186b6c0612bd4901c89a4f1c"
|
||||
integrity sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==
|
||||
dependencies:
|
||||
dom-serializer "^2.0.0"
|
||||
domelementtype "^2.3.0"
|
||||
domhandler "^5.0.1"
|
||||
|
||||
dotenv@^10.0.0:
|
||||
version "10.0.0"
|
||||
resolved "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz"
|
||||
|
|
@ -3035,6 +3078,11 @@ entities@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
|
||||
integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
|
||||
|
||||
entities@^4.2.0:
|
||||
version "4.4.0"
|
||||
resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174"
|
||||
integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==
|
||||
|
||||
entities@^4.3.0:
|
||||
version "4.3.1"
|
||||
resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.1.tgz#c34062a94c865c322f9d67b4384e4169bcede6a4"
|
||||
|
|
@ -3357,6 +3405,11 @@ esutils@^2.0.2:
|
|||
resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
|
||||
integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
|
||||
|
||||
event-target-shim@^5.0.0:
|
||||
version "5.0.1"
|
||||
resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
|
||||
integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
|
||||
|
||||
execa@^5.1.1:
|
||||
version "5.1.1"
|
||||
resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz"
|
||||
|
|
@ -3451,14 +3504,6 @@ fastq@^1.6.0:
|
|||
dependencies:
|
||||
reusify "^1.0.4"
|
||||
|
||||
favecon@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/favecon/-/favecon-1.0.2.tgz#450ae94066900be474b9e408799ecf6e4a65658c"
|
||||
integrity sha512-2jpgF7sMheI/UBGJLzi4JyknC26Bly1txL3rCNNxsY9yHp4lflB3kEYX79zZ/JeSqmQAf8tbktqonVkuj+iwOQ==
|
||||
dependencies:
|
||||
node-fetch "^2.6.1"
|
||||
node-html-parser "^4.0.0"
|
||||
|
||||
fetch-blob@^3.1.2, fetch-blob@^3.1.4:
|
||||
version "3.2.0"
|
||||
resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz"
|
||||
|
|
@ -4788,13 +4833,6 @@ node-domexception@^1.0.0:
|
|||
resolved "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz"
|
||||
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
|
||||
|
||||
node-fetch@^2.6.1:
|
||||
version "2.6.7"
|
||||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
|
||||
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
|
||||
dependencies:
|
||||
whatwg-url "^5.0.0"
|
||||
|
||||
node-fetch@^3.2.8:
|
||||
version "3.2.10"
|
||||
resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-3.2.10.tgz"
|
||||
|
|
@ -4804,12 +4842,12 @@ node-fetch@^3.2.8:
|
|||
fetch-blob "^3.1.4"
|
||||
formdata-polyfill "^4.0.10"
|
||||
|
||||
node-html-parser@^4.0.0:
|
||||
version "4.1.5"
|
||||
resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-4.1.5.tgz#e3ff5b39a098e70de3629c9c79c4e29a3fa5f062"
|
||||
integrity sha512-NLgqUXtftqnBqIjlRjYSaApaqE7TTxfTiH4VqKCjdUJKFOtUzRwney83EHz2qYc0XoxXAkYdmLjENCuZHvsIFg==
|
||||
node-html-parser@^6.1.1:
|
||||
version "6.1.1"
|
||||
resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-6.1.1.tgz#7f38f4427fafc242a22135d9db80c1455e837467"
|
||||
integrity sha512-eYYblUeoMg0nR6cYGM4GRb1XncNa9FXEftuKAU1qyMIr6rXVtNyUKduvzZtkqFqSHVByq2lLjC7WO8tz7VDmnA==
|
||||
dependencies:
|
||||
css-select "^4.1.3"
|
||||
css-select "^5.1.0"
|
||||
he "1.2.0"
|
||||
|
||||
node-releases@^2.0.3:
|
||||
|
|
@ -6503,11 +6541,6 @@ tough-cookie@~2.4.3:
|
|||
psl "^1.1.24"
|
||||
punycode "^1.4.1"
|
||||
|
||||
tr46@~0.0.3:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
|
||||
integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
|
||||
|
||||
trim-newlines@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"
|
||||
|
|
@ -6760,19 +6793,6 @@ web-streams-polyfill@^3.0.3:
|
|||
resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz"
|
||||
integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
|
||||
|
||||
webidl-conversions@^3.0.0:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
|
||||
integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
|
||||
|
||||
whatwg-url@^5.0.0:
|
||||
version "5.0.0"
|
||||
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
|
||||
integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
|
||||
dependencies:
|
||||
tr46 "~0.0.3"
|
||||
webidl-conversions "^3.0.0"
|
||||
|
||||
which-boxed-primitive@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"
|
||||
|
|
|
|||
Loading…
Reference in New Issue