Resolve favicon locally (WIP)

components/common/Favicon.js

@@ -2,27 +2,14 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import styles from './Favicon.module.css';
 
-function getHostName(url) {
-  const match = url.match(/^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:/\n?=]+)/im);
-  return match && match.length > 1 ? match[1] : null;
-}
+function Favicon({ url, ...props }) {
+  const faviconUrl = url ? url : '/default-favicon.png';
 
-function Favicon({ domain, ...props }) {
-  const hostName = domain ? getHostName(domain) : null;
-
-  return hostName ? (
-    <img
-      className={styles.favicon}
-      src={`https://icons.duckduckgo.com/ip3/${hostName}.ico`}
-      height="16"
-      alt=""
-      {...props}
-    />
-  ) : null;
+  return <img className={styles.favicon} src={faviconUrl} height="16" alt="" {...props} />;
 }
 
 Favicon.propTypes = {
-  domain: PropTypes.string,
+  url: PropTypes.string,
 };
 
 export default Favicon;

components/metrics/WebsiteChart.js

@@ -18,7 +18,7 @@ import styles from './WebsiteChart.module.css';
 export default function WebsiteChart({
   websiteId,
   title,
-  domain,
+  favicon,
   stickyHeader = false,
   showLink = false,
   showChart = true,
@@ -81,7 +81,7 @@ export default function WebsiteChart({
 
   return (
     <div className={styles.container}>
-      <WebsiteHeader websiteId={websiteId} title={title} domain={domain} showLink={showLink} />
+      <WebsiteHeader websiteId={websiteId} title={title} favicon={favicon} showLink={showLink} />
       <div className={classNames(styles.header, 'row')}>
         <StickyHeader
           className={classNames(styles.metrics, 'col row')}

components/metrics/WebsiteHeader.js

@@ -11,10 +11,10 @@ import ActiveUsers from './ActiveUsers';
 import Arrow from 'assets/arrow-right.svg';
 import styles from './WebsiteHeader.module.css';
 
-export default function WebsiteHeader({ websiteId, title, domain, showLink = false }) {
+export default function WebsiteHeader({ websiteId, title, favicon, showLink = false }) {
   const header = showLink ? (
     <>
-      <Favicon domain={domain} />
+      <Favicon url={favicon} />
       <Link
         className={styles.titleLink}
         href="/website/[...id]"
@@ -25,7 +25,7 @@ export default function WebsiteHeader({ websiteId, title, domain, showLink = fal
     </>
   ) : (
     <>
-      <Favicon domain={domain} />
+      <Favicon url={favicon} />
       <OverflowText tooltipId={`${websiteId}-title`}>{title}</OverflowText>
     </>
   );

components/pages/TestConsole.js

@@ -106,7 +106,7 @@ export default function TestConsole() {
               <WebsiteChart
                 websiteId={website.website_id}
                 title={website.name}
-                domain={website.domain}
+                favicon={website.favicon}
                 showLink
               />
               <PageHeader>Events</PageHeader>

components/pages/WebsiteDetails.js

@@ -141,7 +141,7 @@ export default function WebsiteDetails({ websiteId }) {
           <WebsiteChart
             websiteId={websiteId}
             title={data.name}
-            domain={data.domain}
+            favicon={data.favicon}
             onDataLoad={handleDataLoad}
             showLink={false}
             stickyHeader

components/pages/WebsiteList.js

@@ -46,13 +46,13 @@ export default function WebsiteList({ websites, showCharts, limit }) {
 
   return (
     <div>
-      {ordered.map(({ website_id, name, domain }, index) =>
+      {ordered.map(({ website_id, name, favicon }, index) =>
         index < limit ? (
           <div key={website_id} className={styles.website}>
             <WebsiteChart
               websiteId={website_id}
               title={name}
-              domain={domain}
+              favicon={favicon}
               showChart={showCharts}
               showLink
             />

components/settings/WebsiteSettings.js

@@ -83,13 +83,13 @@ export default function WebsiteSettings() {
     </ButtonLayout>
   );
 
-  const DetailsLink = ({ website_id, name, domain }) => (
+  const DetailsLink = ({ website_id, name, favicon }) => (
     <Link
       className={styles.detailLink}
       href="/website/[...id]"
       as={`/website/${website_id}/${name}`}
     >
-      <Favicon domain={domain} />
+      <Favicon url={favicon} />
       <OverflowText tooltipId={`${website_id}-name`}>{name}</OverflowText>
     </Link>
   );

db/postgresql/migrations/04_add_favicon_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "website" ADD COLUMN     "favicon" VARCHAR(500);

db/postgresql/schema.prisma

@@ -85,6 +85,7 @@ model website {
   domain       String?    @db.VarChar(500)
   share_id     String?    @unique @db.VarChar(64)
   created_at   DateTime?  @default(now()) @db.Timestamptz(6)
+  favicon      String?    @db.VarChar(500)
   account      account    @relation(fields: [user_id], references: [user_id])
   event        event[]
   pageview     pageview[]

package.json

@@ -71,6 +71,7 @@
     "del": "^6.0.0",
     "detect-browser": "^5.2.0",
     "dotenv": "^10.0.0",
+    "favecon": "^1.0.2",
     "formik": "^2.2.9",
     "fs-extra": "^10.0.1",
     "immer": "^9.0.12",

pages/api/website/index.js

@@ -2,6 +2,22 @@ import { ok, unauthorized, methodNotAllowed, getRandomChars } from 'next-basics'
 import { updateWebsite, createWebsite, getWebsiteById } from 'queries';
 import { useAuth } from 'lib/middleware';
 import { uuid } from 'lib/crypto';
+import favecon from 'favecon';
+
+const getFavicon = async domain => {
+  try {
+    const icons = await favecon.getIcons(`https://${domain}`);
+
+    if (icons.length && icons.length > 0) {
+      return icons[0]?.href;
+    }
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.log(`Could not fetch favicon for domain ${domain}`, e);
+  }
+
+  return null;
+};
 
 export default async (req, res) => {
   await useAuth(req, res);
@@ -13,6 +29,8 @@ export default async (req, res) => {
     const { name, domain, owner } = req.body;
     const website_owner = parseInt(owner);
 
+    const favicon = await getFavicon(domain);
+
     if (website_id) {
       const website = await getWebsiteById(website_id);
 
@@ -28,13 +46,19 @@ export default async (req, res) => {
         share_id = null;
       }
 
-      await updateWebsite(website_id, { name, domain, share_id, user_id: website_owner });
+      await updateWebsite(website_id, { name, domain, share_id, user_id: website_owner, favicon });
 
       return ok(res);
     } else {
       const website_uuid = uuid();
       const share_id = enable_share_url ? getRandomChars(8) : null;
-      const website = await createWebsite(website_owner, { website_uuid, name, domain, share_id });
+      const website = await createWebsite(website_owner, {
+        website_uuid,
+        name,
+        domain,
+        share_id,
+        favicon,
+      });
 
       return ok(res, website);
     }

yarn.lock

@@ -3451,6 +3451,14 @@ fastq@^1.6.0:
   dependencies:
     reusify "^1.0.4"
 
+favecon@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/favecon/-/favecon-1.0.2.tgz#450ae94066900be474b9e408799ecf6e4a65658c"
+  integrity sha512-2jpgF7sMheI/UBGJLzi4JyknC26Bly1txL3rCNNxsY9yHp4lflB3kEYX79zZ/JeSqmQAf8tbktqonVkuj+iwOQ==
+  dependencies:
+    node-fetch "^2.6.1"
+    node-html-parser "^4.0.0"
+
 fetch-blob@^3.1.2, fetch-blob@^3.1.4:
   version "3.2.0"
   resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz"
@@ -3791,6 +3799,11 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+he@1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
+  integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
+
 hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.2:
   version "3.3.2"
   resolved "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz"
@@ -4775,6 +4788,13 @@ node-domexception@^1.0.0:
   resolved "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
+node-fetch@^2.6.1:
+  version "2.6.7"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-fetch@^3.2.8:
   version "3.2.10"
   resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-3.2.10.tgz"
@@ -4784,6 +4804,14 @@ node-fetch@^3.2.8:
     fetch-blob "^3.1.4"
     formdata-polyfill "^4.0.10"
 
+node-html-parser@^4.0.0:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-4.1.5.tgz#e3ff5b39a098e70de3629c9c79c4e29a3fa5f062"
+  integrity sha512-NLgqUXtftqnBqIjlRjYSaApaqE7TTxfTiH4VqKCjdUJKFOtUzRwney83EHz2qYc0XoxXAkYdmLjENCuZHvsIFg==
+  dependencies:
+    css-select "^4.1.3"
+    he "1.2.0"
+
 node-releases@^2.0.3:
   version "2.0.5"
   resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.5.tgz"
@@ -6475,6 +6503,11 @@ tough-cookie@~2.4.3:
     psl "^1.1.24"
     punycode "^1.4.1"
 
+tr46@~0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
+  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
+
 trim-newlines@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"
@@ -6727,6 +6760,19 @@ web-streams-polyfill@^3.0.3:
   resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz"
   integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
 
+webidl-conversions@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
+  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
+
+whatwg-url@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
+  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
+  dependencies:
+    tr46 "~0.0.3"
+    webidl-conversions "^3.0.0"
+
 which-boxed-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"