From 6ba00152c28384bae7f65230c456e28fd34f47eb Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 10 Mar 2023 17:49:19 -0800 Subject: [PATCH 01/15] Updated filter tags. --- components/metrics/FilterTags.js | 16 +++++------ components/metrics/FilterTags.module.css | 21 ++++++++++---- .../pages/settings/teams/TeamWebsitesTable.js | 6 ++-- lib/filters.js | 28 +++---------------- 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/components/metrics/FilterTags.js b/components/metrics/FilterTags.js index 37f5981c..637ea066 100644 --- a/components/metrics/FilterTags.js +++ b/components/metrics/FilterTags.js @@ -32,15 +32,13 @@ export default function FilterTags({ websiteId, params, onClick }) { return null; } return ( -
- +
handleCloseFilter(key)}> + + {`${key}`} = {`${safeDecodeURI(params[key])}`} + + + +
); })} diff --git a/components/metrics/FilterTags.module.css b/components/metrics/FilterTags.module.css index 50ae60a0..1c8458ac 100644 --- a/components/metrics/FilterTags.module.css +++ b/components/metrics/FilterTags.module.css @@ -1,11 +1,22 @@ .filters { display: flex; - justify-content: flex-start; - align-items: flex-start; + align-items: center; + gap: 10px; } .tag { - text-align: center; - margin-bottom: 10px; - margin-right: 20px; + display: flex; + flex-direction: row; + align-items: center; + gap: 10px; + font-size: var(--font-size-sm); + border: 1px solid var(--base600); + border-radius: var(--border-radius); + line-height: 30px; + padding: 0 8px; + cursor: pointer; +} + +.tag:hover { + background: var(--base75); } diff --git a/components/pages/settings/teams/TeamWebsitesTable.js b/components/pages/settings/teams/TeamWebsitesTable.js index a596204c..52141523 100644 --- a/components/pages/settings/teams/TeamWebsitesTable.js +++ b/components/pages/settings/teams/TeamWebsitesTable.js @@ -17,14 +17,14 @@ import { labels } from 'components/messages'; import useUser from 'hooks/useUser'; import useApi from 'hooks/useApi'; -export default function TeamWebsitesTable({ teamId, data = [], onSave }) { +export default function TeamWebsitesTable({ data = [], onSave }) { const { formatMessage } = useIntl(); const { user } = useUser(); const { del, useMutation } = useApi(); - const { mutate } = useMutation(data => del(`/teamWebsites/${data.teamWebsiteId}`)); + const { mutate } = useMutation(({ teamWebsiteId }) => del(`/teamWebsites/${teamWebsiteId}`)); const columns = [ - { name: 'name', label: formatMessage(labels.name), style: { flex: 2 } }, + { name: 'name', label: formatMessage(labels.name) }, { name: 'domain', label: formatMessage(labels.domain) }, { name: 'action', label: ' ' }, ]; diff --git a/lib/filters.js b/lib/filters.js index 7e247220..9681a802 100644 --- a/lib/filters.js +++ b/lib/filters.js @@ -1,30 +1,10 @@ export const urlFilter = data => { - const isValidUrl = url => { - return url !== '' && url !== null; - }; - - const cleanUrl = url => { - try { - const { pathname } = new URL(url, location.origin); - - return pathname; - } catch { - return null; - } - }; - const map = data.reduce((obj, { x, y }) => { - if (!isValidUrl(x)) { - return obj; - } - - const url = cleanUrl(x); - - if (url) { - if (!obj[url]) { - obj[url] = y; + if (x) { + if (!obj[x]) { + obj[x] = y; } else { - obj[url] += y; + obj[x] += y; } } From 3823705fc6810582d56d90bc92142b5ce8a81417 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 14 Mar 2023 11:35:24 -0700 Subject: [PATCH 02/15] Added HoverTooltip component. Removed react-tooltip. --- components/common/HoverTooltip.js | 25 +++++++++++ .../HoverTooltip.module.css} | 2 +- components/common/WorldMap.js | 7 +-- components/metrics/BarChart.js | 45 +++++++++---------- components/metrics/ChartTooltip.js | 26 ----------- components/metrics/EventsChart.js | 1 - components/metrics/PageviewsChart.js | 1 - package.json | 3 +- yarn.lock | 21 ++------- 9 files changed, 56 insertions(+), 75 deletions(-) create mode 100644 components/common/HoverTooltip.js rename components/{metrics/ChartTooltip.module.css => common/HoverTooltip.module.css} (95%) delete mode 100644 components/metrics/ChartTooltip.js diff --git a/components/common/HoverTooltip.js b/components/common/HoverTooltip.js new file mode 100644 index 00000000..3b885b1d --- /dev/null +++ b/components/common/HoverTooltip.js @@ -0,0 +1,25 @@ +import { useEffect, useState } from 'react'; +import { Tooltip } from 'react-basics'; +import styles from './HoverTooltip.module.css'; + +export default function HoverTooltip({ tooltip }) { + const [position, setPosition] = useState({ x: -1000, y: -1000 }); + + useEffect(() => { + const handler = e => { + setPosition({ x: e.clientX, y: e.clientY }); + }; + + document.addEventListener('mousemove', handler); + + return () => { + document.removeEventListener('mousemove', handler); + }; + }, []); + + return ( +
+ +
+ ); +} diff --git a/components/metrics/ChartTooltip.module.css b/components/common/HoverTooltip.module.css similarity index 95% rename from components/metrics/ChartTooltip.module.css rename to components/common/HoverTooltip.module.css index 2d7ba958..519e303d 100644 --- a/components/metrics/ChartTooltip.module.css +++ b/components/common/HoverTooltip.module.css @@ -3,7 +3,7 @@ } .tooltip { - color: var(--msgColor); + position: fixed; pointer-events: none; z-index: 1; } diff --git a/components/common/WorldMap.js b/components/common/WorldMap.js index 8afd84e0..b774702b 100644 --- a/components/common/WorldMap.js +++ b/components/common/WorldMap.js @@ -1,6 +1,5 @@ import { useState, useMemo } from 'react'; import { useRouter } from 'next/router'; -import ReactTooltip from 'react-tooltip'; import { ComposableMap, Geographies, Geography, ZoomableGroup } from 'react-simple-maps'; import classNames from 'classnames'; import { colord } from 'colord'; @@ -9,6 +8,8 @@ import { ISO_COUNTRIES, THEME_COLORS, MAP_FILE } from 'lib/constants'; import styles from './WorldMap.module.css'; import useCountryNames from 'hooks/useCountryNames'; import useLocale from 'hooks/useLocale'; +import HoverTooltip from './HoverTooltip'; +import { formatLongNumber } from '../../lib/format'; function WorldMap({ data, className }) { const { basePath } = useRouter(); @@ -46,7 +47,7 @@ function WorldMap({ data, className }) { function handleHover(code) { if (code === 'AQ') return; const country = data?.find(({ x }) => x === code); - setTooltip(`${countryNames[code]}: ${country?.y || 0} visitors`); + setTooltip(`${countryNames[code]}: ${formatLongNumber(country?.y || 0)} visitors`); } return ( @@ -83,7 +84,7 @@ function WorldMap({ data, className }) { - {tooltip} + {tooltip && }
); } diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js index f70e012e..cab5021c 100644 --- a/components/metrics/BarChart.js +++ b/components/metrics/BarChart.js @@ -1,6 +1,8 @@ import { useState, useRef, useEffect } from 'react'; +import { StatusLight } from 'react-basics'; import classNames from 'classnames'; import ChartJS from 'chart.js'; +import HoverTooltip from 'components/common/HoverTooltip'; import Legend from 'components/metrics/Legend'; import { formatLongNumber } from 'lib/format'; import { dateFormat } from 'lib/date'; @@ -9,10 +11,8 @@ import useTheme from 'hooks/useTheme'; import useForceUpdate from 'hooks/useForceUpdate'; import { DEFAULT_ANIMATION_DURATION, THEME_COLORS } from 'lib/constants'; import styles from './BarChart.module.css'; -import ChartTooltip from './ChartTooltip'; export default function BarChart({ - chartId, datasets, unit, records, @@ -89,22 +89,20 @@ export default function BarChart({ } const [label, value] = body[0].lines[0].split(':'); + const format = unit === 'hour' ? 'EEE p — PPP' : 'PPPP'; - setTooltip({ - title: dateFormat(new Date(+title[0]), getTooltipFormat(unit), locale), - value, - label, - labelColor: labelColors[0].backgroundColor, - }); - } - - function getTooltipFormat(unit) { - switch (unit) { - case 'hour': - return 'EEE p — PPP'; - default: - return 'PPPP'; - } + setTooltip( + <> +
{dateFormat(new Date(+title[0]), format, locale)}
+
+ + + {formatLongNumber(value)} {label} + + +
+ , + ); } function createChart() { @@ -125,6 +123,9 @@ export default function BarChart({ legend: { display: false, }, + onResize: ({ width, height }) => { + //console.log({ width, height }); + }, scales: { xAxes: [ { @@ -206,19 +207,15 @@ export default function BarChart({ updateChart(); } } - }, [datasets, unit, animationDuration, locale, theme]); + }, [datasets, unit, animationDuration, locale]); return ( <> -
+
- + {tooltip && } ); } diff --git a/components/metrics/ChartTooltip.js b/components/metrics/ChartTooltip.js deleted file mode 100644 index c409f462..00000000 --- a/components/metrics/ChartTooltip.js +++ /dev/null @@ -1,26 +0,0 @@ -import { StatusLight } from 'react-basics'; -import styles from './ChartTooltip.module.css'; -import ReactTooltip from 'react-tooltip'; - -export default function ChartTooltip({ chartId, tooltip }) { - if (!tooltip) { - return null; - } - - const { title, value, label, labelColor } = tooltip; - - return ( - -
-
-
{title}
-
- - {value} {label} - -
-
-
- - ); -} diff --git a/components/metrics/EventsChart.js b/components/metrics/EventsChart.js index db647c2f..31d5faa9 100644 --- a/components/metrics/EventsChart.js +++ b/components/metrics/EventsChart.js @@ -76,7 +76,6 @@ export default function EventsChart({ websiteId, className, token }) { return ( Date: Tue, 14 Mar 2023 17:27:17 -0700 Subject: [PATCH 03/15] schema changes to CH, Postgres, MySQL --- db/clickhouse/schema.sql | 21 +++++++++++++------ db/mysql/migrations/01_init/migration.sql | 7 +++++-- db/mysql/schema.prisma | 21 +++++++++++-------- .../08_split_url_referrer/migration.sql | 20 ++++++++++++++++++ db/postgresql/schema.prisma | 21 +++++++++++-------- 5 files changed, 64 insertions(+), 26 deletions(-) create mode 100644 db/postgresql/migrations/08_split_url_referrer/migration.sql diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index d4d7310c..eb59a295 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -19,8 +19,11 @@ CREATE TABLE event subdivision2 LowCardinality(String), city String, --pageview - url String, - referrer String, + url_path String, + url_query String, + referrer_path String, + referrer_query String, + referrer_domain String, page_title String, --event event_type UInt32, @@ -48,8 +51,11 @@ CREATE TABLE event_queue ( subdivision2 LowCardinality(String), city String, --pageview - url String, - referrer String, + url_path String, + url_query String, + referrer_path String, + referrer_query String, + referrer_domain String, page_title String, --event event_type UInt32, @@ -79,8 +85,11 @@ SELECT website_id, subdivision1, subdivision2, city, - url, - referrer, + url_path String, + url_query String, + referrer_path String, + referrer_query String, + referrer_domain String, page_title, event_type, event_name, diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql index a083d03a..2c30d1ed 100644 --- a/db/mysql/migrations/01_init/migration.sql +++ b/db/mysql/migrations/01_init/migration.sql @@ -61,8 +61,11 @@ CREATE TABLE `website_event` ( `website_id` VARCHAR(36) NOT NULL, `session_id` VARCHAR(36) NOT NULL, `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), - `url` VARCHAR(500) NOT NULL, - `referrer` VARCHAR(500) NULL, + `url_path` VARCHAR(500) NOT NULL, + `url_query` VARCHAR(500) NULL, + `referrer_path` VARCHAR(500) NULL, + `referrer_query` VARCHAR(500) NULL, + `referrer_domain` VARCHAR(500) NULL, `page_title` VARCHAR(500) NULL, `event_type` INTEGER UNSIGNED NOT NULL DEFAULT 1, `event_name` VARCHAR(50) NULL, diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index a47ff428..a67aa6e1 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -65,15 +65,18 @@ model Website { } model WebsiteEvent { - id String @id() @map("event_id") @db.VarChar(36) - websiteId String @map("website_id") @db.VarChar(36) - sessionId String @map("session_id") @db.VarChar(36) - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) - url String @db.VarChar(500) - referrer String? @db.VarChar(500) - pageTitle String? @map("page_title") @db.VarChar(500) - eventType Int @default(1) @map("event_type") @db.UnsignedInt - eventName String? @map("event_name") @db.VarChar(50) + id String @id() @map("event_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + sessionId String @map("session_id") @db.VarChar(36) + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) + urlPath String @map("url_path") @db.VarChar(500) + urlQuery String? @map("url_query") @db.VarChar(500) + referrerPath String? @map("referrer_path") @db.VarChar(500) + referrerQuery String? @map("referrer_query") @db.VarChar(500) + referrerDomain String? @map("referrer_domain") @db.VarChar(500) + pageTitle String? @map("page_title") @db.VarChar(500) + eventType Int @default(1) @map("event_type") @db.UnsignedInt + eventName String? @map("event_name") @db.VarChar(50) @@index([createdAt]) @@index([sessionId]) diff --git a/db/postgresql/migrations/08_split_url_referrer/migration.sql b/db/postgresql/migrations/08_split_url_referrer/migration.sql new file mode 100644 index 00000000..c24b0404 --- /dev/null +++ b/db/postgresql/migrations/08_split_url_referrer/migration.sql @@ -0,0 +1,20 @@ +/* + Warnings: + + - You are about to drop the column `userId` on the `team_website` table. All the data in the column will be lost. + - You are about to drop the column `referrer` on the `website_event` table. All the data in the column will be lost. + - You are about to drop the column `url` on the `website_event` table. All the data in the column will be lost. + - Added the required column `url_path` to the `website_event` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "team_website" DROP COLUMN "userId"; + +-- AlterTable +ALTER TABLE "website_event" DROP COLUMN "referrer", +DROP COLUMN "url", +ADD COLUMN "referrer_domain" VARCHAR(500), +ADD COLUMN "referrer_path" VARCHAR(500), +ADD COLUMN "referrer_query" VARCHAR(500), +ADD COLUMN "url_path" VARCHAR(500) NOT NULL, +ADD COLUMN "url_query" VARCHAR(500); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index be3be8f1..82b22c64 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -64,15 +64,18 @@ model Website { } model WebsiteEvent { - id String @id() @map("event_id") @db.Uuid - websiteId String @map("website_id") @db.Uuid - sessionId String @map("session_id") @db.Uuid - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) - url String @db.VarChar(500) - referrer String? @db.VarChar(500) - pageTitle String? @map("page_title") @db.VarChar(500) - eventType Int @default(1) @map("event_type") @db.Integer - eventName String? @map("event_name") @db.VarChar(50) + id String @id() @map("event_id") @db.Uuid + websiteId String @map("website_id") @db.Uuid + sessionId String @map("session_id") @db.Uuid + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + urlPath String @map("url_path") @db.VarChar(500) + urlQuery String? @map("url_query") @db.VarChar(500) + referrerPath String? @map("referrer_path") @db.VarChar(500) + referrerQuery String? @map("referrer_query") @db.VarChar(500) + referrerDomain String? @map("referrer_domain") @db.VarChar(500) + pageTitle String? @map("page_title") @db.VarChar(500) + eventType Int @default(1) @map("event_type") @db.Integer + eventName String? @map("event_name") @db.VarChar(50) @@index([createdAt]) @@index([sessionId]) From cfa36d1802b89cd0d30918fc923c178fd51c79b4 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 14 Mar 2023 17:43:36 -0700 Subject: [PATCH 04/15] clean-up migrations --- db/mysql/migrations/01_init/migration.sql | 4 ---- db/mysql/schema.prisma | 6 ------ db/postgresql/migrations/07_remove_user_id/migration.sql | 3 +-- .../migrations/08_split_url_referrer/migration.sql | 4 ---- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql index 2c30d1ed..eacbc38d 100644 --- a/db/mysql/migrations/01_init/migration.sql +++ b/db/mysql/migrations/01_init/migration.sql @@ -82,14 +82,12 @@ CREATE TABLE `website_event` ( CREATE TABLE `team` ( `team_id` VARCHAR(36) NOT NULL, `name` VARCHAR(50) NOT NULL, - `user_id` VARCHAR(36) NOT NULL, `access_code` VARCHAR(50) NULL, `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), `updated_at` TIMESTAMP(0) NULL, UNIQUE INDEX `team_team_id_key`(`team_id`), UNIQUE INDEX `team_access_code_key`(`access_code`), - INDEX `team_user_id_idx`(`user_id`), INDEX `team_access_code_idx`(`access_code`), PRIMARY KEY (`team_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; @@ -113,13 +111,11 @@ CREATE TABLE `team_user` ( CREATE TABLE `team_website` ( `team_website_id` VARCHAR(36) NOT NULL, `team_id` VARCHAR(36) NOT NULL, - `user_id` VARCHAR(36) NOT NULL, `website_id` VARCHAR(36) NOT NULL, `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), UNIQUE INDEX `team_website_team_website_id_key`(`team_website_id`), INDEX `team_website_team_id_idx`(`team_id`), - INDEX `team_website_user_id_idx`(`user_id`), INDEX `team_website_website_id_idx`(`website_id`), PRIMARY KEY (`team_website_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index a67aa6e1..1f182716 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -19,7 +19,6 @@ model User { teamUser TeamUser[] Website Website[] - teamWebsite TeamWebsite[] @@map("user") } @@ -89,7 +88,6 @@ model WebsiteEvent { model Team { id String @id() @unique() @map("team_id") @db.VarChar(36) name String @db.VarChar(50) - userId String @map("user_id") @db.VarChar(36) accessCode String? @unique @map("access_code") @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) updatedAt DateTime? @map("updated_at") @db.Timestamp(0) @@ -97,7 +95,6 @@ model Team { teamUsers TeamUser[] teamWebsite TeamWebsite[] - @@index([userId]) @@index([accessCode]) @@map("team") } @@ -121,16 +118,13 @@ model TeamUser { model TeamWebsite { id String @id() @unique() @map("team_website_id") @db.VarChar(36) teamId String @map("team_id") @db.VarChar(36) - userId String @map("user_id") @db.VarChar(36) websiteId String @map("website_id") @db.VarChar(36) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) team Team @relation(fields: [teamId], references: [id]) - user User @relation(fields: [userId], references: [id]) website Website @relation(fields: [websiteId], references: [id]) @@index([teamId]) - @@index([userId]) @@index([websiteId]) @@map("team_website") } diff --git a/db/postgresql/migrations/07_remove_user_id/migration.sql b/db/postgresql/migrations/07_remove_user_id/migration.sql index 63122f49..ff92f545 100644 --- a/db/postgresql/migrations/07_remove_user_id/migration.sql +++ b/db/postgresql/migrations/07_remove_user_id/migration.sql @@ -15,5 +15,4 @@ DROP INDEX "team_website_user_id_idx"; ALTER TABLE "team" DROP COLUMN "user_id"; -- AlterTable -ALTER TABLE "team_website" DROP COLUMN "user_id", -ADD COLUMN "userId" UUID; +ALTER TABLE "team_website" DROP COLUMN "user_id"; \ No newline at end of file diff --git a/db/postgresql/migrations/08_split_url_referrer/migration.sql b/db/postgresql/migrations/08_split_url_referrer/migration.sql index c24b0404..7129d87a 100644 --- a/db/postgresql/migrations/08_split_url_referrer/migration.sql +++ b/db/postgresql/migrations/08_split_url_referrer/migration.sql @@ -1,15 +1,11 @@ /* Warnings: - - You are about to drop the column `userId` on the `team_website` table. All the data in the column will be lost. - You are about to drop the column `referrer` on the `website_event` table. All the data in the column will be lost. - You are about to drop the column `url` on the `website_event` table. All the data in the column will be lost. - Added the required column `url_path` to the `website_event` table without a default value. This is not possible if the table is not empty. */ --- AlterTable -ALTER TABLE "team_website" DROP COLUMN "userId"; - -- AlterTable ALTER TABLE "website_event" DROP COLUMN "referrer", DROP COLUMN "url", From 7596f425455d885ee35269bba31cf2d93201d4ac Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 14 Mar 2023 18:59:19 -0700 Subject: [PATCH 05/15] update pageview, event save --- queries/analytics/event/saveEvent.ts | 24 +++++----- queries/analytics/pageview/savePageView.ts | 53 ++++++++++++++++------ 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index 60f5348f..73ecee27 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -8,8 +8,8 @@ import cache from 'lib/cache'; export async function saveEvent(args: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; pageTitle?: string; eventName?: string; hostname?: string; @@ -32,20 +32,20 @@ export async function saveEvent(args: { async function relationalQuery(data: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; pageTitle?: string; eventName?: string; }) { - const { websiteId, id: sessionId, url, eventName, referrer, pageTitle } = data; + const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data; return prisma.client.websiteEvent.create({ data: { id: uuid(), websiteId, sessionId, - url: url?.substring(0, URL_LENGTH), - referrer: referrer?.substring(0, URL_LENGTH), + urlPath: urlPath?.substring(0, URL_LENGTH), + urlQuery: urlQuery?.substring(0, URL_LENGTH), pageTitle: pageTitle, eventType: EVENT_TYPE.customEvent, eventName: eventName?.substring(0, EVENT_NAME_LENGTH), @@ -56,8 +56,8 @@ async function relationalQuery(data: { async function clickhouseQuery(data: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; pageTitle?: string; eventName?: string; hostname?: string; @@ -74,7 +74,8 @@ async function clickhouseQuery(data: { const { websiteId, id: sessionId, - url, + urlPath, + urlQuery, pageTitle, eventName, country, @@ -94,7 +95,8 @@ async function clickhouseQuery(data: { subdivision1: subdivision1 ? subdivision1 : null, subdivision2: subdivision2 ? subdivision2 : null, city: city ? city : null, - url: url?.substring(0, URL_LENGTH), + urlPath: urlPath?.substring(0, URL_LENGTH), + urlQuery: urlQuery?.substring(0, URL_LENGTH), page_title: pageTitle, event_type: EVENT_TYPE.customEvent, event_name: eventName?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/savePageView.ts b/queries/analytics/pageview/savePageView.ts index 24f45b5e..9a2a9c9e 100644 --- a/queries/analytics/pageview/savePageView.ts +++ b/queries/analytics/pageview/savePageView.ts @@ -8,8 +8,11 @@ import { uuid } from 'lib/crypto'; export async function savePageView(args: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; + referrerPath?: string; + referrerQuery?: string; + referrerDomain?: string; pageTitle?: string; hostname?: string; browser?: string; @@ -31,19 +34,34 @@ export async function savePageView(args: { async function relationalQuery(data: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; + referrerPath?: string; + referrerQuery?: string; + referrerDomain?: string; pageTitle?: string; }) { - const { websiteId, id: sessionId, url, referrer, pageTitle } = data; + const { + websiteId, + id: sessionId, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + pageTitle, + } = data; return prisma.client.websiteEvent.create({ data: { id: uuid(), websiteId, sessionId, - url: url?.substring(0, URL_LENGTH), - referrer: referrer?.substring(0, URL_LENGTH), + urlPath: urlPath?.substring(0, URL_LENGTH), + urlQuery: urlQuery?.substring(0, URL_LENGTH), + referrerPath: referrerPath?.substring(0, URL_LENGTH), + referrerQuery: referrerQuery?.substring(0, URL_LENGTH), + referrerDomain: referrerDomain?.substring(0, URL_LENGTH), pageTitle: pageTitle, eventType: EVENT_TYPE.pageView, }, @@ -53,8 +71,11 @@ async function relationalQuery(data: { async function clickhouseQuery(data: { id: string; websiteId: string; - url: string; - referrer?: string; + urlPath: string; + urlQuery?: string; + referrerPath?: string; + referrerQuery?: string; + referrerDomain?: string; pageTitle?: string; hostname?: string; browser?: string; @@ -70,8 +91,11 @@ async function clickhouseQuery(data: { const { websiteId, id: sessionId, - url, - referrer, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, pageTitle, country, subdivision1, @@ -90,8 +114,11 @@ async function clickhouseQuery(data: { subdivision1: subdivision1 ? subdivision1 : null, subdivision2: subdivision2 ? subdivision2 : null, city: city ? city : null, - url: url?.substring(0, URL_LENGTH), - referrer: referrer?.substring(0, URL_LENGTH), + urlPath: urlPath?.substring(0, URL_LENGTH), + urlQuery: urlQuery?.substring(0, URL_LENGTH), + referrerPath: referrerPath?.substring(0, URL_LENGTH), + referrerQuery: referrerQuery?.substring(0, URL_LENGTH), + referrerDomain: referrerDomain?.substring(0, URL_LENGTH), page_title: pageTitle, event_type: EVENT_TYPE.pageView, created_at: getDateFormat(new Date()), From c1d3e9ec6757d4f67265486968e4ef1fc0702df9 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 14 Mar 2023 22:37:50 -0700 Subject: [PATCH 06/15] Upgraded Chart.js to v4. Updated charts. --- components/layout/Header.module.css | 2 +- components/metrics/BarChart.js | 165 ++++++------------ components/metrics/BarChart.module.css | 4 +- components/metrics/Legend.module.css | 2 +- components/metrics/PageviewsChart.js | 49 +++--- hooks/useDateRange.js | 1 - lib/date.js | 8 +- package.json | 5 +- pages/_app.js | 1 + .../analytics/pageview/getPageviewStats.ts | 4 +- yarn.lock | 46 +++-- 11 files changed, 113 insertions(+), 174 deletions(-) diff --git a/components/layout/Header.module.css b/components/layout/Header.module.css index c896e967..e796b6f0 100644 --- a/components/layout/Header.module.css +++ b/components/layout/Header.module.css @@ -12,7 +12,7 @@ gap: 10px; font-size: var(--font-size-lg); font-weight: 700; - color: var(--font-color100); + color: var(--font-color100) !important; } .buttons { diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js index cab5021c..e3e42b7e 100644 --- a/components/metrics/BarChart.js +++ b/components/metrics/BarChart.js @@ -1,7 +1,7 @@ -import { useState, useRef, useEffect } from 'react'; +import { useState, useRef, useEffect, useMemo } from 'react'; import { StatusLight } from 'react-basics'; import classNames from 'classnames'; -import ChartJS from 'chart.js'; +import Chart from 'chart.js/auto'; import HoverTooltip from 'components/common/HoverTooltip'; import Legend from 'components/metrics/Legend'; import { formatLongNumber } from 'lib/format'; @@ -15,7 +15,6 @@ import styles from './BarChart.module.css'; export default function BarChart({ datasets, unit, - records, animationDuration = DEFAULT_ANIMATION_DURATION, className, stacked = false, @@ -30,74 +29,36 @@ export default function BarChart({ const [theme] = useTheme(); const forceUpdate = useForceUpdate(); - const colors = { - text: THEME_COLORS[theme].gray700, - line: THEME_COLORS[theme].gray200, - zeroLine: THEME_COLORS[theme].gray500, - }; - - function renderXLabel(label, index, values) { - if (loading) return ''; - const d = new Date(values[index].value); - const sw = canvas.current.width / window.devicePixelRatio; - - switch (unit) { - case 'minute': - return index % 2 === 0 ? dateFormat(d, 'H:mm', locale) : ''; - case 'hour': - return dateFormat(d, 'p', locale); - case 'day': - if (records > 25) { - if (sw <= 275) { - return index % 10 === 0 ? dateFormat(d, 'M/d', locale) : ''; - } - if (sw <= 550) { - return index % 5 === 0 ? dateFormat(d, 'M/d', locale) : ''; - } - if (sw <= 700) { - return index % 2 === 0 ? dateFormat(d, 'M/d', locale) : ''; - } - return dateFormat(d, 'MMM d', locale); - } - if (sw <= 375) { - return index % 2 === 0 ? dateFormat(d, 'MMM d', locale) : ''; - } - if (sw <= 425) { - return dateFormat(d, 'MMM d', locale); - } - return dateFormat(d, 'EEE M/d', locale); - case 'month': - if (sw <= 330) { - return index % 2 === 0 ? dateFormat(d, 'MMM', locale) : ''; - } - return dateFormat(d, 'MMM', locale); - default: - return label; - } - } + const colors = useMemo( + () => ({ + text: THEME_COLORS[theme].gray700, + line: THEME_COLORS[theme].gray200, + zeroLine: THEME_COLORS[theme].gray500, + }), + [theme], + ); function renderYLabel(label) { return +label > 1000 ? formatLongNumber(label) : label; } function renderTooltip(model) { - const { opacity, title, body, labelColors } = model; + const { opacity, labelColors, dataPoints } = model.tooltip; - if (!opacity || !title) { + if (!dataPoints?.length || !opacity) { setTooltip(null); return; } - const [label, value] = body[0].lines[0].split(':'); const format = unit === 'hour' ? 'EEE p — PPP' : 'PPPP'; setTooltip( <> -
{dateFormat(new Date(+title[0]), format, locale)}
+
{dateFormat(new Date(dataPoints[0].raw.x), format, locale)}
- + - {formatLongNumber(value)} {label} + {formatLongNumber(dataPoints[0].raw.y)} {dataPoints[0].dataset.label}
@@ -107,68 +68,53 @@ export default function BarChart({ function createChart() { const options = { + responsive: true, + maintainAspectRatio: false, animation: { duration: animationDuration, + resize: { + duration: 0, + }, + active: { + duration: 0, + }, }, - tooltips: { - enabled: false, - custom: renderTooltip, - }, - hover: { - animationDuration: 0, - }, - responsive: true, - responsiveAnimationDuration: 0, - maintainAspectRatio: false, - legend: { - display: false, - }, - onResize: ({ width, height }) => { - //console.log({ width, height }); + plugins: { + legend: { + display: false, + }, + tooltip: { + enabled: false, + external: renderTooltip, + }, }, scales: { - xAxes: [ - { - type: 'time', - distribution: 'series', - time: { - unit, - tooltipFormat: 'x', - }, - ticks: { - callback: renderXLabel, - minRotation: 0, - maxRotation: 0, - fontColor: colors.text, - autoSkipPadding: 1, - }, - gridLines: { - display: false, - }, - offset: true, - stacked: true, + x: { + type: 'time', + stacked: true, + grid: { + display: false, }, - ], - yAxes: [ - { - ticks: { - callback: renderYLabel, - beginAtZero: true, - fontColor: colors.text, - }, - gridLines: { - color: colors.line, - zeroLineColor: colors.zeroLine, - }, - stacked, + ticks: { + autoSkip: false, + maxRotation: 0, }, - ], + }, + y: { + type: 'linear', + min: 0, + beginAtZero: true, + stacked, + ticks: { + callback: renderYLabel, + }, + }, }, }; onCreate(options); - chart.current = new ChartJS(canvas.current, { + chart.current = new Chart(canvas.current, { type: 'bar', data: { datasets, @@ -180,16 +126,7 @@ export default function BarChart({ function updateChart() { const { options } = chart.current; - options.legend.labels.fontColor = colors.text; - options.scales.xAxes[0].time.unit = unit; - options.scales.xAxes[0].ticks.callback = renderXLabel; - options.scales.xAxes[0].ticks.fontColor = colors.text; - options.scales.yAxes[0].ticks.fontColor = colors.text; - options.scales.yAxes[0].ticks.precision = 0; - options.scales.yAxes[0].gridLines.color = colors.line; - options.scales.yAxes[0].gridLines.zeroLineColor = colors.zeroLine; options.animation.duration = animationDuration; - options.tooltips.custom = renderTooltip; onUpdate(chart.current); @@ -207,7 +144,7 @@ export default function BarChart({ updateChart(); } } - }, [datasets, unit, animationDuration, locale]); + }, [datasets, unit, animationDuration, locale, loading]); return ( <> diff --git a/components/metrics/BarChart.module.css b/components/metrics/BarChart.module.css index 593bff91..1768d15b 100644 --- a/components/metrics/BarChart.module.css +++ b/components/metrics/BarChart.module.css @@ -1,10 +1,12 @@ .chart { position: relative; height: 400px; + + overflow: hidden; } @media only screen and (max-width: 992px) { .chart { - height: 200px; + /*height: 200px;*/ } } diff --git a/components/metrics/Legend.module.css b/components/metrics/Legend.module.css index e78bf609..20deffcd 100644 --- a/components/metrics/Legend.module.css +++ b/components/metrics/Legend.module.css @@ -8,7 +8,7 @@ .label { display: flex; align-items: center; - font-size: var(--font-size-xs); + font-size: var(--font-size-sm); cursor: pointer; } diff --git a/components/metrics/PageviewsChart.js b/components/metrics/PageviewsChart.js index 8161c910..38550969 100644 --- a/components/metrics/PageviewsChart.js +++ b/components/metrics/PageviewsChart.js @@ -25,12 +25,16 @@ export default function PageviewsChart({ const primaryColor = colord(THEME_COLORS[theme].primary); return { views: { - background: primaryColor.alpha(0.4).toRgbString(), - border: primaryColor.alpha(0.5).toRgbString(), + hoverBackgroundColor: primaryColor.alpha(0.7).toRgbString(), + backgroundColor: primaryColor.alpha(0.4).toRgbString(), + borderColor: primaryColor.alpha(0.7).toRgbString(), + hoverBorderColor: primaryColor.toRgbString(), }, visitors: { - background: primaryColor.alpha(0.6).toRgbString(), - border: primaryColor.alpha(0.7).toRgbString(), + hoverBackgroundColor: primaryColor.alpha(0.9).toRgbString(), + backgroundColor: primaryColor.alpha(0.6).toRgbString(), + borderColor: primaryColor.alpha(0.9).toRgbString(), + hoverBorderColor: primaryColor.toRgbString(), }, }; }, [theme]); @@ -50,29 +54,30 @@ export default function PageviewsChart({ return null; } + const datasets = [ + { + label: formatMessage(labels.uniqueVisitors), + data: data.sessions, + lineTension: 0, + borderWidth: 1, + ...colors.visitors, + }, + { + label: formatMessage(labels.pageViews), + data: data.pageviews, + lineTension: 0, + borderWidth: 1, + ...colors.views, + }, + ]; + return (
{ - return normalize(getDateFromString(e.t)).getTime() === t.getTime(); + const d = data.find(({ x }) => { + return normalize(getDateFromString(x)).getTime() === t.getTime(); }); - return x?.y || 0; + return d?.y || 0; } for (let i = 0; i < n; i++) { const t = normalize(add(startDate, i)); const y = findData(t); - arr.push({ ...data[i], t, y }); + arr.push({ x: t, y }); } return arr; diff --git a/package.json b/package.json index ed7d6e73..5153f6cc 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,8 @@ "@umami/prisma-client": "^0.2.0", "@umami/redis-client": "^0.2.0", "chalk": "^4.1.1", - "chart.js": "^2.9.4", + "chart.js": "^4.2.1", + "chartjs-adapter-date-fns": "^3.0.0", "classnames": "^2.3.1", "clickhouse": "^2.5.0", "colord": "^2.9.2", @@ -93,7 +94,7 @@ "node-fetch": "^3.2.8", "npm-run-all": "^4.1.5", "react": "^18.2.0", - "react-basics": "^0.72.0", + "react-basics": "^0.73.0", "react-beautiful-dnd": "^13.1.0", "react-dom": "^18.2.0", "react-intl": "^5.24.7", diff --git a/pages/_app.js b/pages/_app.js index ec14d084..ad9e71ac 100644 --- a/pages/_app.js +++ b/pages/_app.js @@ -10,6 +10,7 @@ import 'styles/locale.css'; import 'styles/index.css'; import '@fontsource/inter/400.css'; import '@fontsource/inter/700.css'; +import 'chartjs-adapter-date-fns'; import Script from 'next/script'; const client = new QueryClient({ diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index b2d86b33..273151aa 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -50,7 +50,7 @@ async function relationalQuery( const { filterQuery, joinSession } = parseFilters(filters, params); return rawQuery( - `select ${getDateQuery('website_event.created_at', unit, timezone)} t, + `select ${getDateQuery('website_event.created_at', unit, timezone)} x, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from website_event ${joinSession} @@ -83,7 +83,7 @@ async function clickhouseQuery( return rawQuery( `select - ${getDateStringQuery('g.t', unit)} as t, + ${getDateStringQuery('g.t', unit)} as x, g.y as y from (select diff --git a/yarn.lock b/yarn.lock index 397c8249..ed85f811 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1745,6 +1745,11 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@kurkle/color@^0.3.0": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@kurkle/color/-/color-0.3.2.tgz#5acd38242e8bde4f9986e7913c8fdf49d3aa199f" + integrity sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw== + "@netlify/esbuild-android-64@0.14.39": version "0.14.39" resolved "https://registry.yarnpkg.com/@netlify/esbuild-android-64/-/esbuild-android-64-0.14.39.tgz#7bd30aba94a92351d2c5e25e178ceb824f3c2f99" @@ -3348,28 +3353,17 @@ chalk@^4.0.0, chalk@^4.1.1, chalk@^4.1.2: ansi-styles "^4.1.0" supports-color "^7.1.0" -chart.js@^2.9.4: - version "2.9.4" - resolved "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz" - integrity sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A== +chart.js@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-4.2.1.tgz#d2bd5c98e9a0ae35408975b638f40513b067ba1d" + integrity sha512-6YbpQ0nt3NovAgOzbkSSeeAQu/3za1319dPUQTXn9WcOpywM8rGKxJHrhS8V8xEkAlk8YhEfjbuAPfUyp6jIsw== dependencies: - chartjs-color "^2.1.0" - moment "^2.10.2" + "@kurkle/color" "^0.3.0" -chartjs-color-string@^0.6.0: - version "0.6.0" - resolved "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz" - integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== - dependencies: - color-name "^1.0.0" - -chartjs-color@^2.1.0: - version "2.4.1" - resolved "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz" - integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== - dependencies: - chartjs-color-string "^0.6.0" - color-convert "^1.9.3" +chartjs-adapter-date-fns@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chartjs-adapter-date-fns/-/chartjs-adapter-date-fns-3.0.0.tgz#c25f63c7f317c1f96f9a7c44bd45eeedb8a478e5" + integrity sha512-Rs3iEB3Q5pJ973J93OBTpnP7qoGwvq3nUnoMdtxO+9aoJof7UFcRbWcIDteXuYd1fgAvct/32T9qaLyLuZVwCg== chokidar@^3.5.3: version "3.5.3" @@ -3464,7 +3458,7 @@ cluster-key-slot@^1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== -color-convert@^1.9.0, color-convert@^1.9.3: +color-convert@^1.9.0: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -6117,7 +6111,7 @@ moment-timezone@^0.5.35: dependencies: moment "^2.29.4" -"moment@>= 2.9.0", moment@^2.10.2, moment@^2.29.4: +"moment@>= 2.9.0", moment@^2.29.4: version "2.29.4" resolved "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== @@ -7082,10 +7076,10 @@ rc@^1.2.7: minimist "^1.2.0" strip-json-comments "~2.0.1" -react-basics@^0.72.0: - version "0.72.0" - resolved "https://registry.yarnpkg.com/react-basics/-/react-basics-0.72.0.tgz#23dbd61d5ac6bb8b8d61f1f3adcebb7edeab8a26" - integrity sha512-dWthEwyh/ilt1BSPYwMdd1oE/OFDp8oZ5udZGrdXs5guRh9Ukar4V4chQPbnuZPYUnAH4jg19H5Fesvz2lSaaw== +react-basics@^0.73.0: + version "0.73.0" + resolved "https://registry.yarnpkg.com/react-basics/-/react-basics-0.73.0.tgz#9555563f3407ac417dc833dfca47588123d55535" + integrity sha512-eEK8yWWrXO7JATBlPKBfFQlD1hNZoNeEtlYNx+QjOCLKu1qjClutP5nXWHmX4gHE97XFwUKzbTU35NkNEy5C0w== dependencies: classnames "^2.3.1" date-fns "^2.29.3" From be8eb61f7fa56453cda073608ba94b32a4793162 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 14 Mar 2023 23:29:53 -0700 Subject: [PATCH 07/15] Fixed issue with responsive charts. --- components/metrics/BarChart.module.css | 1 - components/metrics/Legend.module.css | 2 +- components/metrics/WebsiteChart.module.css | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/components/metrics/BarChart.module.css b/components/metrics/BarChart.module.css index 1768d15b..c54f2d4a 100644 --- a/components/metrics/BarChart.module.css +++ b/components/metrics/BarChart.module.css @@ -1,7 +1,6 @@ .chart { position: relative; height: 400px; - overflow: hidden; } diff --git a/components/metrics/Legend.module.css b/components/metrics/Legend.module.css index 20deffcd..b079e67f 100644 --- a/components/metrics/Legend.module.css +++ b/components/metrics/Legend.module.css @@ -2,7 +2,7 @@ display: flex; justify-content: center; flex-wrap: wrap; - margin-top: 10px; + padding: 10px 0; } .label { diff --git a/components/metrics/WebsiteChart.module.css b/components/metrics/WebsiteChart.module.css index 05bfcbb8..3a3d4718 100644 --- a/components/metrics/WebsiteChart.module.css +++ b/components/metrics/WebsiteChart.module.css @@ -7,7 +7,7 @@ .chart { position: relative; - padding-bottom: 10px; + overflow: hidden; } .title { From 275e7d3df616f17987dfc37e09465f1ec3d21b6b Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 15 Mar 2023 16:06:51 -0700 Subject: [PATCH 08/15] update send --- pages/api/send.ts | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/pages/api/send.ts b/pages/api/send.ts index 04e62f07..abee3a9b 100644 --- a/pages/api/send.ts +++ b/pages/api/send.ts @@ -34,9 +34,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { const { type, payload } = getJsonBody(req); - const { referrer, eventName, pageTitle } = payload; - let { url } = payload; - + const { url, referrer, eventName, pageTitle } = payload; const ignoreIps = process.env.IGNORE_IP; const ignoreHostnames = process.env.IGNORE_HOSTNAME; @@ -83,17 +81,41 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { const session = req.session; + let urlPath = url.split('?')[0]; + const urlQuery = url.split('?')[1]; + let referrerPath; + let referrerQuery; + let referrerDomain; + + if (referrer.substring(0, 4) === 'http') { + const newRef = new URL(referrer); + referrerPath = newRef.pathname; + referrerDomain = newRef.hostname; + referrerQuery = newRef.search.substring(1); + } else { + referrerPath = referrer.split('?')[0]; + referrerQuery = referrer.split('?')[1]; + } + if (process.env.REMOVE_TRAILING_SLASH) { - url = url.replace(/\/$/, ''); + urlPath = urlPath.replace(/\/$/, ''); } if (type === 'pageview') { - await savePageView({ ...session, url, referrer, pageTitle }); + await savePageView({ + ...session, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + pageTitle, + }); } else if (type === 'event') { await saveEvent({ ...session, - url, - referrer, + urlPath, + urlQuery, pageTitle, eventName, }); From 54051d7204c086367189bf271e0682c87e930a81 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Wed, 15 Mar 2023 16:27:05 -0700 Subject: [PATCH 09/15] Updated tracker script name and endpoint. --- rollup.tracker.config.js | 4 ++-- tracker/index.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rollup.tracker.config.js b/rollup.tracker.config.js index 5bc09610..f4e7223c 100644 --- a/rollup.tracker.config.js +++ b/rollup.tracker.config.js @@ -6,12 +6,12 @@ import { terser } from 'rollup-plugin-terser'; export default { input: 'tracker/index.js', output: { - file: 'public/umami.js', + file: 'public/script.js', format: 'iife', }, plugins: [ replace({ - '/api/collect': process.env.COLLECT_API_ENDPOINT || '/api/collect', + '/api/send': process.env.COLLECT_API_ENDPOINT || '/api/send', delimiters: ['', ''], preventAssignment: true, }), diff --git a/tracker/index.js b/tracker/index.js index baff16ec..8f27ab36 100644 --- a/tracker/index.js +++ b/tracker/index.js @@ -61,7 +61,7 @@ const root = hostUrl ? hostUrl.replace(/\/$/, '') : currentScript.src.split('/').slice(0, -1).join('/'); - const endpoint = `${root}/api/collect`; + const endpoint = `${root}/api/send`; const screen = `${width}x${height}`; const eventClass = /^umami--([a-z]+)--([\w]+[\w-]*)$/; const eventSelect = "[class*='umami--']"; From f569f6ac916ca9b58538a6b1af885e1f1f3ebe25 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 15 Mar 2023 16:39:55 -0700 Subject: [PATCH 10/15] fix null ip lookup error --- lib/detect.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/detect.js b/lib/detect.js index d8294f05..e5bdd733 100644 --- a/lib/detect.js +++ b/lib/detect.js @@ -79,10 +79,10 @@ export async function getClientInfo(req, { screen }) { const userAgent = req.headers['user-agent']; const ip = getIpAddress(req); const location = await getLocation(ip); - const country = location.country; - const subdivision1 = location.subdivision1; - const subdivision2 = location.subdivision2; - const city = location.city; + const country = location?.country; + const subdivision1 = location?.subdivision1; + const subdivision2 = location?.subdivision2; + const city = location?.city; const browser = browserName(userAgent); const os = detectOS(userAgent); const device = getDevice(screen, browser, os); From afec08b35558e6f5d648b150a1054f5212ead916 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 16 Mar 2023 02:05:56 -0700 Subject: [PATCH 11/15] Fixed grid lines on bar chart. --- components/metrics/BarChart.js | 49 +++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js index e3e42b7e..fb71df97 100644 --- a/components/metrics/BarChart.js +++ b/components/metrics/BarChart.js @@ -8,7 +8,6 @@ import { formatLongNumber } from 'lib/format'; import { dateFormat } from 'lib/date'; import useLocale from 'hooks/useLocale'; import useTheme from 'hooks/useTheme'; -import useForceUpdate from 'hooks/useForceUpdate'; import { DEFAULT_ANIMATION_DURATION, THEME_COLORS } from 'lib/constants'; import styles from './BarChart.module.css'; @@ -27,22 +26,20 @@ export default function BarChart({ const [tooltip, setTooltip] = useState(null); const { locale } = useLocale(); const [theme] = useTheme(); - const forceUpdate = useForceUpdate(); const colors = useMemo( () => ({ text: THEME_COLORS[theme].gray700, line: THEME_COLORS[theme].gray200, - zeroLine: THEME_COLORS[theme].gray500, }), [theme], ); - function renderYLabel(label) { + const renderYLabel = label => { return +label > 1000 ? formatLongNumber(label) : label; - } + }; - function renderTooltip(model) { + const renderTooltip = model => { const { opacity, labelColors, dataPoints } = model.tooltip; if (!dataPoints?.length || !opacity) { @@ -64,9 +61,11 @@ export default function BarChart({
, ); - } + }; + + const createChart = () => { + Chart.defaults.font.family = 'Inter'; - function createChart() { const options = { responsive: true, maintainAspectRatio: false, @@ -92,10 +91,17 @@ export default function BarChart({ x: { type: 'time', stacked: true, + time: { + unit, + }, grid: { display: false, }, + border: { + color: colors.line, + }, ticks: { + color: colors.text, autoSkip: false, maxRotation: 0, }, @@ -105,7 +111,14 @@ export default function BarChart({ min: 0, beginAtZero: true, stacked, + grid: { + color: colors.line, + }, + border: { + color: colors.line, + }, ticks: { + color: colors.text, callback: renderYLabel, }, }, @@ -121,19 +134,23 @@ export default function BarChart({ }, options, }); - } + }; - function updateChart() { - const { options } = chart.current; + const updateChart = () => { + const { animation, scales } = chart.current.options; - options.animation.duration = animationDuration; + animation.duration = animationDuration; + scales.x.ticks.color = colors.text; + scales.x.time.unit = unit; + scales.x.border.color = colors.line; + scales.y.ticks.color = colors.text; + scales.y.grid.color = colors.line; + scales.y.border.color = colors.line; onUpdate(chart.current); chart.current.update(); - - forceUpdate(); - } + }; useEffect(() => { if (datasets) { @@ -144,7 +161,7 @@ export default function BarChart({ updateChart(); } } - }, [datasets, unit, animationDuration, locale, loading]); + }, [datasets, unit, theme, animationDuration, locale, loading]); return ( <> From 9c8cb8247edb5263db97f0f43bbba9f929451558 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 16 Mar 2023 10:36:30 -0700 Subject: [PATCH 12/15] update gitignore to new script name --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 54410324..566dc61b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ # production /build -/public/umami.js +/public/script.js /public/geo # misc From c62e2e9cc71c2da9e2fdb8b94884d72ae529e5c9 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 16 Mar 2023 15:56:05 -0700 Subject: [PATCH 13/15] Update tooltip date formats. --- components/metrics/BarChart.js | 82 +++++++++++++----------- components/metrics/BarChart.module.css | 10 +++ components/metrics/EventsChart.js | 4 +- components/metrics/PageviewsChart.js | 2 - components/pages/realtime/RealtimeLog.js | 4 +- 5 files changed, 60 insertions(+), 42 deletions(-) diff --git a/components/metrics/BarChart.js b/components/metrics/BarChart.js index fb71df97..65881e28 100644 --- a/components/metrics/BarChart.js +++ b/components/metrics/BarChart.js @@ -1,4 +1,4 @@ -import { useState, useRef, useEffect, useMemo } from 'react'; +import { useState, useRef, useEffect, useMemo, useCallback } from 'react'; import { StatusLight } from 'react-basics'; import classNames from 'classnames'; import Chart from 'chart.js/auto'; @@ -15,14 +15,14 @@ export default function BarChart({ datasets, unit, animationDuration = DEFAULT_ANIMATION_DURATION, - className, stacked = false, loading = false, onCreate = () => {}, onUpdate = () => {}, + className, }) { const canvas = useRef(); - const chart = useRef(); + const chart = useRef(null); const [tooltip, setTooltip] = useState(null); const { locale } = useLocale(); const [theme] = useTheme(); @@ -39,34 +39,45 @@ export default function BarChart({ return +label > 1000 ? formatLongNumber(label) : label; }; - const renderTooltip = model => { - const { opacity, labelColors, dataPoints } = model.tooltip; + const renderTooltip = useCallback( + model => { + const { opacity, labelColors, dataPoints } = model.tooltip; - if (!dataPoints?.length || !opacity) { - setTooltip(null); - return; - } + if (!dataPoints?.length || !opacity) { + setTooltip(null); + return; + } - const format = unit === 'hour' ? 'EEE p — PPP' : 'PPPP'; + const formats = { + millisecond: 'T', + second: 'pp', + minute: 'p', + hour: 'h aaa', + day: 'PPPP', + week: 'PPPP', + month: 'LLLL yyyy', + quarter: 'qqq', + year: 'yyyy', + }; - setTooltip( - <> -
{dateFormat(new Date(dataPoints[0].raw.x), format, locale)}
-
- - - {formatLongNumber(dataPoints[0].raw.y)} {dataPoints[0].dataset.label} - - -
- , - ); - }; + setTooltip( +
+
{dateFormat(new Date(dataPoints[0].raw.x), formats[unit], locale)}
+
+ +
+ {formatLongNumber(dataPoints[0].raw.y)} {dataPoints[0].dataset.label} +
+ +
+
, + ); + }, + [unit], + ); - const createChart = () => { - Chart.defaults.font.family = 'Inter'; - - const options = { + const getOptions = useCallback(() => { + return { responsive: true, maintainAspectRatio: false, animation: { @@ -124,6 +135,12 @@ export default function BarChart({ }, }, }; + }, [animationDuration, renderTooltip, stacked, colors]); + + const createChart = () => { + Chart.defaults.font.family = 'Inter'; + + const options = getOptions(); onCreate(options); @@ -137,15 +154,9 @@ export default function BarChart({ }; const updateChart = () => { - const { animation, scales } = chart.current.options; + setTooltip(null); - animation.duration = animationDuration; - scales.x.ticks.color = colors.text; - scales.x.time.unit = unit; - scales.x.border.color = colors.line; - scales.y.ticks.color = colors.text; - scales.y.grid.color = colors.line; - scales.y.border.color = colors.line; + chart.current.options = getOptions(); onUpdate(chart.current); @@ -157,7 +168,6 @@ export default function BarChart({ if (!chart.current) { createChart(); } else { - setTooltip(null); updateChart(); } } diff --git a/components/metrics/BarChart.module.css b/components/metrics/BarChart.module.css index c54f2d4a..850d1ea7 100644 --- a/components/metrics/BarChart.module.css +++ b/components/metrics/BarChart.module.css @@ -4,6 +4,16 @@ overflow: hidden; } +.tooltip { + display: flex; + flex-direction: column; + gap: 10px; +} + +.tooltip .value { + text-transform: lowercase; +} + @media only screen and (max-width: 992px) { .chart { /*height: 200px;*/ diff --git a/components/metrics/EventsChart.js b/components/metrics/EventsChart.js index 31d5faa9..6ccab9ab 100644 --- a/components/metrics/EventsChart.js +++ b/components/metrics/EventsChart.js @@ -33,12 +33,12 @@ export default function EventsChart({ websiteId, className, token }) { if (!data) return []; if (isLoading) return data; - const map = data.reduce((obj, { x, t, y }) => { + const map = data.reduce((obj, { x, y }) => { if (!obj[x]) { obj[x] = []; } - obj[x].push({ t, y }); + obj[x].push({ x, y }); return obj; }, {}); diff --git a/components/metrics/PageviewsChart.js b/components/metrics/PageviewsChart.js index 38550969..47a085f0 100644 --- a/components/metrics/PageviewsChart.js +++ b/components/metrics/PageviewsChart.js @@ -58,14 +58,12 @@ export default function PageviewsChart({ { label: formatMessage(labels.uniqueVisitors), data: data.sessions, - lineTension: 0, borderWidth: 1, ...colors.visitors, }, { label: formatMessage(labels.pageViews), data: data.pageviews, - lineTension: 0, borderWidth: 1, ...colors.views, }, diff --git a/components/pages/realtime/RealtimeLog.js b/components/pages/realtime/RealtimeLog.js index 06b7537e..be7aeb8d 100644 --- a/components/pages/realtime/RealtimeLog.js +++ b/components/pages/realtime/RealtimeLog.js @@ -5,7 +5,7 @@ import { FixedSizeList } from 'react-window'; import firstBy from 'thenby'; import FilterButtons from 'components/common/FilterButtons'; import NoData from 'components/common/NoData'; -import { getDeviceMessage, labels, messages } from 'components/messages'; +import { labels, messages } from 'components/messages'; import useLocale from 'hooks/useLocale'; import useCountryNames from 'hooks/useCountryNames'; import { BROWSERS } from 'lib/constants'; @@ -102,7 +102,7 @@ export default function RealtimeLog({ data, websiteDomain }) { country: {countryNames[country] || formatMessage(labels.unknown)}, browser: {BROWSERS[browser]}, os: {os}, - device: {formatMessage(getDeviceMessage(device))}, + device: {formatMessage(labels[device] || labels.unknown)}, }} /> ); From b0c5899569148765eb1eb3cbb2c0453b9161903f Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 20 Mar 2023 11:26:45 -0700 Subject: [PATCH 14/15] update prisma / ch filters logic --- components/metrics/MetricsTable.js | 2 +- db/clickhouse/schema.sql | 10 +++++----- lib/clickhouse.ts | 15 ++++++++++----- lib/prisma.ts | 15 ++++++++++----- pages/api/send.ts | 4 ++-- pages/api/websites/[id]/metrics.ts | 27 +++++++++++++++++++++------ 6 files changed, 49 insertions(+), 24 deletions(-) diff --git a/components/metrics/MetricsTable.js b/components/metrics/MetricsTable.js index 258ea1e8..945f7796 100644 --- a/components/metrics/MetricsTable.js +++ b/components/metrics/MetricsTable.js @@ -40,7 +40,7 @@ export default function MetricsTable({ const { data, isLoading, isFetched, error } = useQuery( [ - 'websites:mnetrics', + 'websites:metrics', { websiteId, type, modified, url, referrer, os, browser, device, country }, ], () => diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index eb59a295..c0a2f62d 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -85,11 +85,11 @@ SELECT website_id, subdivision1, subdivision2, city, - url_path String, - url_query String, - referrer_path String, - referrer_query String, - referrer_domain String, + url_path, + url_query, + referrer_path, + referrer_query, + referrer_domain, page_title, event_type, event_name, diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index a3030eeb..073a92e7 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -74,6 +74,9 @@ function getFilterQuery(filters = {}, params = {}) { switch (key) { case 'url': + arr.push(`and url_path = {${key}:String}`); + params[key] = filter; + break; case 'pageTitle': case 'os': case 'browser': @@ -92,18 +95,20 @@ function getFilterQuery(filters = {}, params = {}) { break; case 'referrer': - arr.push(`and referrer ILIKE {${key}:String}`); - params[key] = `%${filter}`; + arr.push(`and referrer_domain= {${key}:String}`); + params[key] = filter; break; case 'domain': - arr.push(`and referrer NOT ILIKE {${key}:String}`); - arr.push(`and referrer NOT ILIKE '/%'`); + arr.push(`and referrer_domain NOT ILIKE {${key}:String}`); + arr.push(`and referrer_domain NOT ILIKE '/%'`); params[key] = `%://${filter}/%`; break; case 'query': - arr.push(`and url like '%?%'`); + arr.push(`and url_query= {${key}:String}`); + params[key] = filter; + break; } return arr; diff --git a/lib/prisma.ts b/lib/prisma.ts index 4461f044..20a5e4a6 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -74,6 +74,9 @@ function getFilterQuery(filters = {}, params = []): string { switch (key) { case 'url': + arr.push(`and url_path=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); + break; case 'os': case 'pageTitle': case 'browser': @@ -92,18 +95,20 @@ function getFilterQuery(filters = {}, params = []): string { break; case 'referrer': - arr.push(`and referrer like $${params.length + 1}`); - params.push(`%${decodeURIComponent(filter)}%`); + arr.push(`and referrer_domain=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); break; case 'domain': - arr.push(`and referrer not like $${params.length + 1}`); - arr.push(`and referrer not like '/%'`); + arr.push(`and referrer_domain not like $${params.length + 1}`); + arr.push(`and referrer_domain not like '/%'`); params.push(`%://${filter}/%`); break; case 'query': - arr.push(`and url like '%?%'`); + arr.push(`and url_query=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); + break; } return arr; diff --git a/pages/api/send.ts b/pages/api/send.ts index abee3a9b..86632bf3 100644 --- a/pages/api/send.ts +++ b/pages/api/send.ts @@ -87,12 +87,12 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { let referrerQuery; let referrerDomain; - if (referrer.substring(0, 4) === 'http') { + try { const newRef = new URL(referrer); referrerPath = newRef.pathname; referrerDomain = newRef.hostname; referrerQuery = newRef.search.substring(1); - } else { + } catch { referrerPath = referrer.split('?')[0]; referrerQuery = referrer.split('?')[1]; } diff --git a/pages/api/websites/[id]/metrics.ts b/pages/api/websites/[id]/metrics.ts index c547daee..ad57790e 100644 --- a/pages/api/websites/[id]/metrics.ts +++ b/pages/api/websites/[id]/metrics.ts @@ -6,7 +6,17 @@ import { NextApiResponse } from 'next'; import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics'; import { getPageviewMetrics, getSessionMetrics, getWebsite } from 'queries'; -const sessionColumns = ['browser', 'os', 'device', 'screen', 'country', 'language']; +const sessionColumns = [ + 'browser', + 'os', + 'device', + 'screen', + 'country', + 'language', + 'subdivision1', + 'subdivision2', + 'city', +]; const pageviewColumns = ['url', 'referrer', 'query', 'pageTitle']; function getTable(type) { @@ -26,12 +36,17 @@ function getTable(type) { } function getColumn(type) { - if (type === 'event') { - return 'event_name'; - } - if (type === 'query') { - return 'url'; + switch (type) { + case 'url': + return 'url_path'; + case 'referrer': + return 'referrer_domain'; + case 'event': + return 'event_name'; + case 'query': + return 'url_query'; } + return type; } From 611169c65f3c316126fb1f5fa074802bbe2d1a22 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Mon, 20 Mar 2023 13:39:40 -0700 Subject: [PATCH 15/15] Fix referrers table. --- components/metrics/ReferrersTable.js | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/components/metrics/ReferrersTable.js b/components/metrics/ReferrersTable.js index afbdecc0..7af102e7 100644 --- a/components/metrics/ReferrersTable.js +++ b/components/metrics/ReferrersTable.js @@ -1,29 +1,11 @@ -import { useState } from 'react'; import { useIntl } from 'react-intl'; import MetricsTable from './MetricsTable'; -import FilterButtons from 'components/common/FilterButtons'; import FilterLink from 'components/common/FilterLink'; -import { refFilter } from 'lib/filters'; import { labels } from 'components/messages'; -import { FILTER_COMBINED, FILTER_RAW } from 'lib/constants'; -const filters = { - [FILTER_RAW]: null, - [FILTER_COMBINED]: refFilter, -}; - -export default function ReferrersTable({ websiteId, showFilters, ...props }) { - const [filter, setFilter] = useState(FILTER_COMBINED); +export default function ReferrersTable({ websiteId, ...props }) { const { formatMessage } = useIntl(); - const items = [ - { - label: formatMessage(labels.filterCombined), - key: FILTER_COMBINED, - }, - { label: formatMessage(labels.filterRaw), key: FILTER_RAW }, - ]; - const renderLink = ({ w: link, x: referrer }) => { return referrer ? ( @@ -34,14 +16,12 @@ export default function ReferrersTable({ websiteId, showFilters, ...props }) { return ( <> - {showFilters && }