ambiser
/
umami
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish event_data relational.

pull/1841/head
Brian Cao 2023-03-23 12:28:57 -07:00
parent f1f602b42d
commit 2343949d3e
12 changed files with 133 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
components/pages/console/TestConsole.js
View File

@ -30,7 +30,16 @@ export default function TestConsole() {
window.umami.trackEvent('track-event-with-data', {
test: 'test-data',
time: new Date(),
number: 1,
time2: new Date().toISOString(),
nested: {
test: 'test-data',
number: 1,
object: {
test: 'test-data',
},
},
array: [1, 2, 3],
});
}
@ -49,7 +58,7 @@ export default function TestConsole() {
async
defer
data-website-id={website.id}
src={`${basePath}/umami.js`}
src={`${basePath}/script.js`}
data-cache="true"
/>
)}

13
db/mysql/migrations/01_init/migration.sql
View File

@ -83,23 +83,17 @@ CREATE TABLE `event_data` (
`event_id` VARCHAR(36) NOT NULL,
`website_event_id` VARCHAR(36) NOT NULL,
`website_id` VARCHAR(36) NOT NULL,
`session_id` VARCHAR(36) NOT NULL,
`url_path` VARCHAR(500) NOT NULL,
`event_name` VARCHAR(500) NOT NULL,
`event_key` VARCHAR(500) NOT NULL,
`event_string_value` VARCHAR(500) NOT NULL,
`event_numeric_value` DECIMAL(19, 4) NOT NULL,
`event_string_value` VARCHAR(500) NULL,
`event_numeric_value` DECIMAL(19, 4) NULL,
`event_date_value` TIMESTAMP(0) NULL,
`event_data_type` INTEGER UNSIGNED NOT NULL,
`created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
INDEX `event_data_created_at_idx`(`created_at`),
INDEX `event_data_session_id_idx`(`session_id`),
INDEX `event_data_website_id_idx`(`website_id`),
INDEX `event_data_website_event_id_idx`(`website_event_id`),
INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`),
INDEX `event_data_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
INDEX `event_data_website_id_session_id_website_event_id_created_at_idx`(`website_id`, `session_id`, `website_event_id`, `created_at`),
PRIMARY KEY (`event_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
@ -144,6 +138,3 @@ CREATE TABLE `team_website` (
INDEX `team_website_website_id_idx`(`website_id`),
PRIMARY KEY (`team_website_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- AddSystemUser
INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');

14
db/mysql/schema.prisma
View File

@ -39,7 +39,6 @@ model Session {
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
websiteEvent WebsiteEvent[]
eventData EventData[]
@@index([createdAt])
@@index([websiteId])
@ -96,27 +95,20 @@ model EventData {
id String @id() @map("event_id") @db.VarChar(36)
websiteEventId String @map("website_event_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
sessionId String @map("session_id") @db.VarChar(36)
urlPath String @map("url_path") @db.VarChar(500)
eventName String @map("event_name") @db.VarChar(500)
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal @map("event_numeric_value") @db.Decimal(19,4)
eventStringValue String? @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)
eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0)
eventDataType Int @map("event_data_type") @db.UnsignedInt
createdAt DateTime? @default(now()) @map("created_at")@db.Timestamp(0)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
website Website @relation(fields: [websiteId], references: [id])
websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([sessionId])
@@index([websiteId])
@@index([websiteEventId])
@@index([websiteId, websiteEventId, createdAt])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, sessionId, websiteEventId, createdAt])
@@map("event_data")
}

38
db/postgresql/migrations/09_add_new_event_data/migration.sql
View File

@ -1,38 +0,0 @@
-- CreateTable
CREATE TABLE "event_data" (
"event_id" UUID NOT NULL,
"website_event_id" UUID NOT NULL,
"website_id" UUID NOT NULL,
"session_id" UUID NOT NULL,
"url_path" VARCHAR(500) NOT NULL,
"event_name" VARCHAR(500) NOT NULL,
"event_key" VARCHAR(500) NOT NULL,
"event_string_value" VARCHAR(500) NOT NULL,
"event_numeric_value" DECIMAL(19,4) NOT NULL,
"event_date_value" TIMESTAMPTZ(6),
"event_data_type" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id")
);
-- CreateIndex
CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at");
-- CreateIndex
CREATE INDEX "event_data_session_id_idx" ON "event_data"("session_id");
-- CreateIndex
CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id");
-- CreateIndex
CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id");
-- CreateIndex
CREATE INDEX "event_data_website_id_website_event_id_created_at_idx" ON "event_data"("website_id", "website_event_id", "created_at");
-- CreateIndex
CREATE INDEX "event_data_website_id_session_id_created_at_idx" ON "event_data"("website_id", "session_id", "created_at");
-- CreateIndex
CREATE INDEX "event_data_website_id_session_id_website_event_id_created_a_idx" ON "event_data"("website_id", "session_id", "website_event_id", "created_at");

23
db/postgresql/migrations/09_event_data/migration.sql Normal file
View File

@ -0,0 +1,23 @@
-- CreateTable
CREATE TABLE "event_data" (
"event_id" UUID NOT NULL,
"website_id" UUID NOT NULL,
"website_event_id" UUID NOT NULL,
"event_key" VARCHAR(500) NOT NULL,
"event_string_value" VARCHAR(500),
"event_numeric_value" DECIMAL(19,4),
"event_date_value" TIMESTAMPTZ(6),
"event_data_type" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id")
);
-- CreateIndex
CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at");
-- CreateIndex
CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id");
-- CreateIndex
CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id");

17
db/postgresql/schema.prisma
View File

@ -39,7 +39,6 @@ model Session {
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
websiteEvent WebsiteEvent[]
eventData EventData[]
@@index([createdAt])
@@index([websiteId])
@ -59,7 +58,7 @@ model Website {
user User? @relation(fields: [userId], references: [id])
teamWebsite TeamWebsite[]
eventData EventData[]
EventData EventData[]
@@index([userId])
@@index([createdAt])
@ -94,29 +93,21 @@ model WebsiteEvent {
model EventData {
id String @id() @map("event_id") @db.Uuid
websiteEventId String @map("website_event_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
urlPath String @map("url_path") @db.VarChar(500)
eventName String @map("event_name") @db.VarChar(500)
websiteEventId String @map("website_event_id") @db.Uuid
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String @map("event_string_value") @db.VarChar(500)
eventNumericValue Int @map("event_numeric_value") @db.Integer
eventStringValue String? @map("event_string_value") @db.VarChar(500)
eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)
eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6)
eventDataType Int @map("event_data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([sessionId])
@@index([websiteId])
@@index([websiteEventId])
@@index([websiteId, websiteEventId, createdAt])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, sessionId, websiteEventId, createdAt])
@@map("event_data")
}

19
lib/prisma.ts
View File

@ -72,33 +72,34 @@ function getEventDataFilterQuery(
}[],
params: any[],
) {
const query = filters.reduce((ac, cv, i) => {
const type = getEventDataType(cv);
const query = filters.reduce((ac, cv) => {
const type = getEventDataType(cv.eventValue);
let value = cv.eventValue;
ac.push(`and (event_key = $${params.length + 1}`);
params.push(cv.eventKey);
switch (type) {
case 'number':
ac.push(`and event_numeric_value = {${cv.eventKey}${i}:UInt64}`);
params.push(cv.eventValue);
ac.push(`and event_numeric_value = $${params.length + 1})`);
params.push(value);
break;
case 'string':
ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
ac.push(`and event_string_value = $${params.length + 1})`);
params.push(decodeURIComponent(cv.eventValue as string));
break;
case 'boolean':
ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
ac.push(`and event_string_value = $${params.length + 1})`);
params.push(decodeURIComponent(cv.eventValue as string));
value = cv ? 'true' : 'false';
break;
case 'date':
ac.push(`and event_date_value = {${cv.eventKey}${i}:DateTime('UTC')}`);
ac.push(`and event_date_value = $${params.length + 1})`);
params.push(cv.eventValue);
break;
}
params[`${cv.eventKey}${i}`] = value;
return ac;
}, []);

4
pages/api/send.ts
View File

@ -99,8 +99,8 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
referrerDomain = newRef.hostname;
referrerQuery = newRef.search.substring(1);
} catch {
referrerPath = referrer.split('?')[0];
referrerQuery = referrer.split('?')[1];
referrerPath = referrer?.split('?')[0];
referrerQuery = referrer?.split('?')[1];
}
if (process.env.REMOVE_TRAILING_SLASH) {

22
queries/analytics/event/saveEvent.ts
View File

@ -40,11 +40,13 @@ async function relationalQuery(data: {
eventName?: string;
eventData?: any;
}) {
const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data;
const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data;
const website = await cache.fetchWebsite(websiteId);
const websiteEventId = uuid();
return prisma.client.websiteEvent.create({
const websiteEvent = prisma.client.websiteEvent.create({
data: {
id: uuid(),
id: websiteEventId,
websiteId,
sessionId,
urlPath: urlPath?.substring(0, URL_LENGTH),
@ -54,6 +56,20 @@ async function relationalQuery(data: {
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
},
});
if (eventData) {
await saveEventData({
websiteId,
sessionId,
eventId: websiteEventId,
revId: website?.revId,
urlPath: urlPath?.substring(0, URL_LENGTH),
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
eventData,
});
}
return websiteEvent;
}
async function clickhouseQuery(data: {

7
queries/analytics/event/getEventData.ts → queries/analytics/eventData/getEventData.ts
View File

@ -48,7 +48,7 @@ async function relationalQuery(
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
const params: any = [websiteId, startDate, endDate, eventName];
const params: any = [websiteId, startDate, endDate, eventName || ''];
return rawQuery(
`select
@ -59,6 +59,11 @@ async function relationalQuery(
timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
}
from event_data
${
eventName || urlPath
? 'join website_event on event_data.id = website_event.website_event_id'
: ''
}
where website_id = $1${toUuid()}
and created_at between $2 and $3
${eventName ? `and eventName = $4` : ''}

55
queries/analytics/eventData/saveEventData.ts
View File

@ -1,18 +1,21 @@
import { Prisma } from '@prisma/client';
import { EVENT_DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { flattenJSON } from 'lib/eventData';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import { EventData } from 'lib/types';
export async function saveEventData(args: {
websiteId: string;
sessionId: string;
eventId: string;
revId: number;
urlPath: string;
eventName: string;
sessionId?: string;
revId?: number;
urlPath?: string;
eventName?: string;
eventData: EventData;
createdAt: string;
createdAt?: string;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
@ -22,25 +25,44 @@ export async function saveEventData(args: {
async function relationalQuery(data: {
websiteId: string;
sessionId: string;
eventId: string;
revId: number;
eventName: string;
eventData: EventData;
createdAt: string;
}) {
return data;
}): Promise<Prisma.BatchPayload> {
const { websiteId, eventId, eventData } = data;
const jsonKeys = flattenJSON(eventData);
//id, websiteEventId, eventStringValue
const flattendData = jsonKeys.map(a => ({
id: uuid(),
websiteEventId: eventId,
websiteId,
eventKey: a.key,
eventStringValue:
a.eventDataType === EVENT_DATA_TYPE.string ||
a.eventDataType === EVENT_DATA_TYPE.boolean ||
a.eventDataType === EVENT_DATA_TYPE.array
? a.value
: null,
eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,
eventDataType: a.eventDataType,
}));
return prisma.client.eventData.createMany({
data: flattendData,
});
}
async function clickhouseQuery(data: {
websiteId: string;
sessionId: string;
eventId: string;
revId: number;
urlPath: string;
eventName: string;
sessionId?: string;
revId?: number;
urlPath?: string;
eventName?: string;
eventData: EventData;
createdAt: string;
createdAt?: string;
}) {
const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data;
@ -67,7 +89,6 @@ async function clickhouseQuery(data: {
event_data_type: a.eventDataType,
created_at: createdAt,
}));
``;
await sendMessages(messages, 'event_data');

2
queries/index.js
View File

@ -4,7 +4,7 @@ export * from './admin/user';
export * from './admin/website';
export * from './analytics/event/getEventMetrics';
export * from './analytics/event/getEvents';
export * from './analytics/event/getEventData';
export * from './analytics/eventData/getEventData';
export * from './analytics/event/saveEvent';
export * from './analytics/pageview/getPageviewMetrics';
export * from './analytics/pageview/getPageviews';