From 2343949d3e815a09c3f2a5f9abb99d21937f5369 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 23 Mar 2023 12:28:57 -0700 Subject: [PATCH] Finish event_data relational. --- components/pages/console/TestConsole.js | 11 +++- db/mysql/migrations/01_init/migration.sql | 13 +---- db/mysql/schema.prisma | 42 ++++++-------- .../09_add_new_event_data/migration.sql | 38 ------------- .../migrations/09_event_data/migration.sql | 23 ++++++++ db/postgresql/schema.prisma | 19 ++----- lib/prisma.ts | 19 ++++--- pages/api/send.ts | 4 +- queries/analytics/event/saveEvent.ts | 22 +++++++- .../{event => eventData}/getEventData.ts | 7 ++- queries/analytics/eventData/saveEventData.ts | 55 +++++++++++++------ queries/index.js | 2 +- 12 files changed, 133 insertions(+), 122 deletions(-) delete mode 100644 db/postgresql/migrations/09_add_new_event_data/migration.sql create mode 100644 db/postgresql/migrations/09_event_data/migration.sql rename queries/analytics/{event => eventData}/getEventData.ts (93%) diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.js index 28e723ef..c7e41c62 100644 --- a/components/pages/console/TestConsole.js +++ b/components/pages/console/TestConsole.js @@ -30,7 +30,16 @@ export default function TestConsole() { window.umami.trackEvent('track-event-with-data', { test: 'test-data', time: new Date(), + number: 1, time2: new Date().toISOString(), + nested: { + test: 'test-data', + number: 1, + object: { + test: 'test-data', + }, + }, + array: [1, 2, 3], }); } @@ -49,7 +58,7 @@ export default function TestConsole() { async defer data-website-id={website.id} - src={`${basePath}/umami.js`} + src={`${basePath}/script.js`} data-cache="true" /> )} diff --git a/db/mysql/migrations/01_init/migration.sql b/db/mysql/migrations/01_init/migration.sql index 586c604f..0e52ce55 100644 --- a/db/mysql/migrations/01_init/migration.sql +++ b/db/mysql/migrations/01_init/migration.sql @@ -83,23 +83,17 @@ CREATE TABLE `event_data` ( `event_id` VARCHAR(36) NOT NULL, `website_event_id` VARCHAR(36) NOT NULL, `website_id` VARCHAR(36) NOT NULL, - `session_id` VARCHAR(36) NOT NULL, - `url_path` VARCHAR(500) NOT NULL, - `event_name` VARCHAR(500) NOT NULL, `event_key` VARCHAR(500) NOT NULL, - `event_string_value` VARCHAR(500) NOT NULL, - `event_numeric_value` DECIMAL(19, 4) NOT NULL, + `event_string_value` VARCHAR(500) NULL, + `event_numeric_value` DECIMAL(19, 4) NULL, `event_date_value` TIMESTAMP(0) NULL, `event_data_type` INTEGER UNSIGNED NOT NULL, `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), INDEX `event_data_created_at_idx`(`created_at`), - INDEX `event_data_session_id_idx`(`session_id`), INDEX `event_data_website_id_idx`(`website_id`), INDEX `event_data_website_event_id_idx`(`website_event_id`), INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`), - INDEX `event_data_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`), - INDEX `event_data_website_id_session_id_website_event_id_created_at_idx`(`website_id`, `session_id`, `website_event_id`, `created_at`), PRIMARY KEY (`event_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; @@ -144,6 +138,3 @@ CREATE TABLE `team_website` ( INDEX `team_website_website_id_idx`(`website_id`), PRIMARY KEY (`team_website_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; - --- AddSystemUser -INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa'); \ No newline at end of file diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 7a7a1b15..13dcdcbc 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -17,29 +17,28 @@ model User { updatedAt DateTime? @map("updated_at") @db.Timestamp(0) deletedAt DateTime? @map("deleted_at") @db.Timestamp(0) - website Website[] - teamUser TeamUser[] + website Website[] + teamUser TeamUser[] @@map("user") } model Session { - id String @id @unique @map("session_id") @db.VarChar(36) - websiteId String @map("website_id") @db.VarChar(36) - hostname String? @db.VarChar(100) - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - device String? @db.VarChar(20) - screen String? @db.VarChar(11) - language String? @db.VarChar(35) - country String? @db.Char(2) - subdivision1 String? @db.Char(3) - subdivision2 String? @db.VarChar(50) - city String? @db.VarChar(50) + id String @id @unique @map("session_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + hostname String? @db.VarChar(100) + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + device String? @db.VarChar(20) + screen String? @db.VarChar(11) + language String? @db.VarChar(35) + country String? @db.Char(2) + subdivision1 String? @db.Char(3) + subdivision2 String? @db.VarChar(50) + city String? @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) websiteEvent WebsiteEvent[] - eventData EventData[] @@index([createdAt]) @@index([websiteId]) @@ -96,27 +95,20 @@ model EventData { id String @id() @map("event_id") @db.VarChar(36) websiteEventId String @map("website_event_id") @db.VarChar(36) websiteId String @map("website_id") @db.VarChar(36) - sessionId String @map("session_id") @db.VarChar(36) - urlPath String @map("url_path") @db.VarChar(500) - eventName String @map("event_name") @db.VarChar(500) eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String @map("event_string_value") @db.VarChar(500) - eventNumericValue Decimal @map("event_numeric_value") @db.Decimal(19,4) + eventStringValue String? @map("event_string_value") @db.VarChar(500) + eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0) eventDataType Int @map("event_data_type") @db.UnsignedInt - createdAt DateTime? @default(now()) @map("created_at")@db.Timestamp(0) + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) - session Session @relation(fields: [sessionId], references: [id]) @@index([createdAt]) - @@index([sessionId]) @@index([websiteId]) @@index([websiteEventId]) @@index([websiteId, websiteEventId, createdAt]) - @@index([websiteId, sessionId, createdAt]) - @@index([websiteId, sessionId, websiteEventId, createdAt]) @@map("event_data") } diff --git a/db/postgresql/migrations/09_add_new_event_data/migration.sql b/db/postgresql/migrations/09_add_new_event_data/migration.sql deleted file mode 100644 index 932d4f19..00000000 --- a/db/postgresql/migrations/09_add_new_event_data/migration.sql +++ /dev/null @@ -1,38 +0,0 @@ --- CreateTable -CREATE TABLE "event_data" ( - "event_id" UUID NOT NULL, - "website_event_id" UUID NOT NULL, - "website_id" UUID NOT NULL, - "session_id" UUID NOT NULL, - "url_path" VARCHAR(500) NOT NULL, - "event_name" VARCHAR(500) NOT NULL, - "event_key" VARCHAR(500) NOT NULL, - "event_string_value" VARCHAR(500) NOT NULL, - "event_numeric_value" DECIMAL(19,4) NOT NULL, - "event_date_value" TIMESTAMPTZ(6), - "event_data_type" INTEGER NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id") -); - --- CreateIndex -CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at"); - --- CreateIndex -CREATE INDEX "event_data_session_id_idx" ON "event_data"("session_id"); - --- CreateIndex -CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id"); - --- CreateIndex -CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id"); - --- CreateIndex -CREATE INDEX "event_data_website_id_website_event_id_created_at_idx" ON "event_data"("website_id", "website_event_id", "created_at"); - --- CreateIndex -CREATE INDEX "event_data_website_id_session_id_created_at_idx" ON "event_data"("website_id", "session_id", "created_at"); - --- CreateIndex -CREATE INDEX "event_data_website_id_session_id_website_event_id_created_a_idx" ON "event_data"("website_id", "session_id", "website_event_id", "created_at"); diff --git a/db/postgresql/migrations/09_event_data/migration.sql b/db/postgresql/migrations/09_event_data/migration.sql new file mode 100644 index 00000000..1196e1c7 --- /dev/null +++ b/db/postgresql/migrations/09_event_data/migration.sql @@ -0,0 +1,23 @@ +-- CreateTable +CREATE TABLE "event_data" ( + "event_id" UUID NOT NULL, + "website_id" UUID NOT NULL, + "website_event_id" UUID NOT NULL, + "event_key" VARCHAR(500) NOT NULL, + "event_string_value" VARCHAR(500), + "event_numeric_value" DECIMAL(19,4), + "event_date_value" TIMESTAMPTZ(6), + "event_data_type" INTEGER NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id") +); + +-- CreateIndex +CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at"); + +-- CreateIndex +CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id"); + +-- CreateIndex +CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 2d21f966..e8b7ee83 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -39,7 +39,6 @@ model Session { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) websiteEvent WebsiteEvent[] - eventData EventData[] @@index([createdAt]) @@index([websiteId]) @@ -59,7 +58,7 @@ model Website { user User? @relation(fields: [userId], references: [id]) teamWebsite TeamWebsite[] - eventData EventData[] + EventData EventData[] @@index([userId]) @@index([createdAt]) @@ -94,29 +93,21 @@ model WebsiteEvent { model EventData { id String @id() @map("event_id") @db.Uuid - websiteEventId String @map("website_event_id") @db.Uuid websiteId String @map("website_id") @db.Uuid - sessionId String @map("session_id") @db.Uuid - urlPath String @map("url_path") @db.VarChar(500) - eventName String @map("event_name") @db.VarChar(500) + websiteEventId String @map("website_event_id") @db.Uuid eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String @map("event_string_value") @db.VarChar(500) - eventNumericValue Int @map("event_numeric_value") @db.Integer + eventStringValue String? @map("event_string_value") @db.VarChar(500) + eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6) eventDataType Int @map("event_data_type") @db.Integer createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) - session Session @relation(fields: [sessionId], references: [id]) @@index([createdAt]) - @@index([sessionId]) @@index([websiteId]) @@index([websiteEventId]) - @@index([websiteId, websiteEventId, createdAt]) - @@index([websiteId, sessionId, createdAt]) - @@index([websiteId, sessionId, websiteEventId, createdAt]) @@map("event_data") } @@ -162,4 +153,4 @@ model TeamWebsite { @@index([teamId]) @@index([websiteId]) @@map("team_website") -} \ No newline at end of file +} diff --git a/lib/prisma.ts b/lib/prisma.ts index ec7c6527..98bf921e 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -72,33 +72,34 @@ function getEventDataFilterQuery( }[], params: any[], ) { - const query = filters.reduce((ac, cv, i) => { - const type = getEventDataType(cv); + const query = filters.reduce((ac, cv) => { + const type = getEventDataType(cv.eventValue); let value = cv.eventValue; + ac.push(`and (event_key = $${params.length + 1}`); + params.push(cv.eventKey); + switch (type) { case 'number': - ac.push(`and event_numeric_value = {${cv.eventKey}${i}:UInt64}`); - params.push(cv.eventValue); + ac.push(`and event_numeric_value = $${params.length + 1})`); + params.push(value); break; case 'string': - ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`); + ac.push(`and event_string_value = $${params.length + 1})`); params.push(decodeURIComponent(cv.eventValue as string)); break; case 'boolean': - ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`); + ac.push(`and event_string_value = $${params.length + 1})`); params.push(decodeURIComponent(cv.eventValue as string)); value = cv ? 'true' : 'false'; break; case 'date': - ac.push(`and event_date_value = {${cv.eventKey}${i}:DateTime('UTC')}`); + ac.push(`and event_date_value = $${params.length + 1})`); params.push(cv.eventValue); break; } - params[`${cv.eventKey}${i}`] = value; - return ac; }, []); diff --git a/pages/api/send.ts b/pages/api/send.ts index eab9fdfd..1a880a46 100644 --- a/pages/api/send.ts +++ b/pages/api/send.ts @@ -99,8 +99,8 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { referrerDomain = newRef.hostname; referrerQuery = newRef.search.substring(1); } catch { - referrerPath = referrer.split('?')[0]; - referrerQuery = referrer.split('?')[1]; + referrerPath = referrer?.split('?')[0]; + referrerQuery = referrer?.split('?')[1]; } if (process.env.REMOVE_TRAILING_SLASH) { diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index aec2a8c4..d1f124ea 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -40,11 +40,13 @@ async function relationalQuery(data: { eventName?: string; eventData?: any; }) { - const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data; + const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data; + const website = await cache.fetchWebsite(websiteId); + const websiteEventId = uuid(); - return prisma.client.websiteEvent.create({ + const websiteEvent = prisma.client.websiteEvent.create({ data: { - id: uuid(), + id: websiteEventId, websiteId, sessionId, urlPath: urlPath?.substring(0, URL_LENGTH), @@ -54,6 +56,20 @@ async function relationalQuery(data: { eventName: eventName?.substring(0, EVENT_NAME_LENGTH), }, }); + + if (eventData) { + await saveEventData({ + websiteId, + sessionId, + eventId: websiteEventId, + revId: website?.revId, + urlPath: urlPath?.substring(0, URL_LENGTH), + eventName: eventName?.substring(0, EVENT_NAME_LENGTH), + eventData, + }); + } + + return websiteEvent; } async function clickhouseQuery(data: { diff --git a/queries/analytics/event/getEventData.ts b/queries/analytics/eventData/getEventData.ts similarity index 93% rename from queries/analytics/event/getEventData.ts rename to queries/analytics/eventData/getEventData.ts index bae64db2..1c34a733 100644 --- a/queries/analytics/event/getEventData.ts +++ b/queries/analytics/eventData/getEventData.ts @@ -48,7 +48,7 @@ async function relationalQuery( ) { const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; - const params: any = [websiteId, startDate, endDate, eventName]; + const params: any = [websiteId, startDate, endDate, eventName || '']; return rawQuery( `select @@ -59,6 +59,11 @@ async function relationalQuery( timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : '' } from event_data + ${ + eventName || urlPath + ? 'join website_event on event_data.id = website_event.website_event_id' + : '' + } where website_id = $1${toUuid()} and created_at between $2 and $3 ${eventName ? `and eventName = $4` : ''} diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index e6998b2c..c7ccb1b1 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -1,18 +1,21 @@ +import { Prisma } from '@prisma/client'; import { EVENT_DATA_TYPE } from 'lib/constants'; +import { uuid } from 'lib/crypto'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { flattenJSON } from 'lib/eventData'; import kafka from 'lib/kafka'; +import prisma from 'lib/prisma'; import { EventData } from 'lib/types'; export async function saveEventData(args: { websiteId: string; - sessionId: string; eventId: string; - revId: number; - urlPath: string; - eventName: string; + sessionId?: string; + revId?: number; + urlPath?: string; + eventName?: string; eventData: EventData; - createdAt: string; + createdAt?: string; }) { return runQuery({ [PRISMA]: () => relationalQuery(args), @@ -22,25 +25,44 @@ export async function saveEventData(args: { async function relationalQuery(data: { websiteId: string; - sessionId: string; eventId: string; - revId: number; - eventName: string; eventData: EventData; - createdAt: string; -}) { - return data; +}): Promise { + const { websiteId, eventId, eventData } = data; + + const jsonKeys = flattenJSON(eventData); + + //id, websiteEventId, eventStringValue + const flattendData = jsonKeys.map(a => ({ + id: uuid(), + websiteEventId: eventId, + websiteId, + eventKey: a.key, + eventStringValue: + a.eventDataType === EVENT_DATA_TYPE.string || + a.eventDataType === EVENT_DATA_TYPE.boolean || + a.eventDataType === EVENT_DATA_TYPE.array + ? a.value + : null, + eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null, + eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null, + eventDataType: a.eventDataType, + })); + + return prisma.client.eventData.createMany({ + data: flattendData, + }); } async function clickhouseQuery(data: { websiteId: string; - sessionId: string; eventId: string; - revId: number; - urlPath: string; - eventName: string; + sessionId?: string; + revId?: number; + urlPath?: string; + eventName?: string; eventData: EventData; - createdAt: string; + createdAt?: string; }) { const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data; @@ -67,7 +89,6 @@ async function clickhouseQuery(data: { event_data_type: a.eventDataType, created_at: createdAt, })); - ``; await sendMessages(messages, 'event_data'); diff --git a/queries/index.js b/queries/index.js index 2d1931ee..d98f2609 100644 --- a/queries/index.js +++ b/queries/index.js @@ -4,7 +4,7 @@ export * from './admin/user'; export * from './admin/website'; export * from './analytics/event/getEventMetrics'; export * from './analytics/event/getEvents'; -export * from './analytics/event/getEventData'; +export * from './analytics/eventData/getEventData'; export * from './analytics/event/saveEvent'; export * from './analytics/pageview/getPageviewMetrics'; export * from './analytics/pageview/getPageviews';