Finish event_data relational.

components/pages/console/TestConsole.js

@@ -30,7 +30,16 @@ export default function TestConsole() {
     window.umami.trackEvent('track-event-with-data', {
       test: 'test-data',
       time: new Date(),
+      number: 1,
       time2: new Date().toISOString(),
+      nested: {
+        test: 'test-data',
+        number: 1,
+        object: {
+          test: 'test-data',
+        },
+      },
+      array: [1, 2, 3],
     });
   }
 
@@ -49,7 +58,7 @@ export default function TestConsole() {
             async
             defer
             data-website-id={website.id}
-            src={`${basePath}/umami.js`}
+            src={`${basePath}/script.js`}
             data-cache="true"
           />
         )}

db/mysql/migrations/01_init/migration.sql

@@ -83,23 +83,17 @@ CREATE TABLE `event_data` (
     `event_id` VARCHAR(36) NOT NULL,
     `website_event_id` VARCHAR(36) NOT NULL,
     `website_id` VARCHAR(36) NOT NULL,
-    `session_id` VARCHAR(36) NOT NULL,
-    `url_path` VARCHAR(500) NOT NULL,
-    `event_name` VARCHAR(500) NOT NULL,
     `event_key` VARCHAR(500) NOT NULL,
-    `event_string_value` VARCHAR(500) NOT NULL,
+    `event_string_value` VARCHAR(500) NULL,
-    `event_numeric_value` DECIMAL(19, 4) NOT NULL,
+    `event_numeric_value` DECIMAL(19, 4) NULL,
     `event_date_value` TIMESTAMP(0) NULL,
     `event_data_type` INTEGER UNSIGNED NOT NULL,
     `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
 
     INDEX `event_data_created_at_idx`(`created_at`),
-    INDEX `event_data_session_id_idx`(`session_id`),
     INDEX `event_data_website_id_idx`(`website_id`),
     INDEX `event_data_website_event_id_idx`(`website_event_id`),
     INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`),
-    INDEX `event_data_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
-    INDEX `event_data_website_id_session_id_website_event_id_created_at_idx`(`website_id`, `session_id`, `website_event_id`, `created_at`),
     PRIMARY KEY (`event_id`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 
@@ -144,6 +138,3 @@ CREATE TABLE `team_website` (
     INDEX `team_website_website_id_idx`(`website_id`),
     PRIMARY KEY (`team_website_id`)
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-
--- AddSystemUser
-INSERT INTO "user" (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');

db/mysql/schema.prisma

@@ -17,29 +17,28 @@ model User {
   updatedAt DateTime? @map("updated_at") @db.Timestamp(0)
   deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
 
-  website     Website[]
+  website  Website[]
-  teamUser    TeamUser[]
+  teamUser TeamUser[]
 
   @@map("user")
 }
 
 model Session {
-  id           String  @id @unique @map("session_id") @db.VarChar(36)
+  id           String    @id @unique @map("session_id") @db.VarChar(36)
-  websiteId    String  @map("website_id") @db.VarChar(36)
+  websiteId    String    @map("website_id") @db.VarChar(36)
-  hostname     String? @db.VarChar(100)
+  hostname     String?   @db.VarChar(100)
-  browser      String? @db.VarChar(20)
+  browser      String?   @db.VarChar(20)
-  os           String? @db.VarChar(20)
+  os           String?   @db.VarChar(20)
-  device       String? @db.VarChar(20)
+  device       String?   @db.VarChar(20)
-  screen       String? @db.VarChar(11)
+  screen       String?   @db.VarChar(11)
-  language     String? @db.VarChar(35)
+  language     String?   @db.VarChar(35)
-  country      String? @db.Char(2)
+  country      String?   @db.Char(2)
-  subdivision1 String? @db.Char(3)
+  subdivision1 String?   @db.Char(3)
-  subdivision2 String? @db.VarChar(50)
+  subdivision2 String?   @db.VarChar(50)
-  city         String? @db.VarChar(50)
+  city         String?   @db.VarChar(50)
   createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
 
   websiteEvent WebsiteEvent[]
-  eventData    EventData[]
 
   @@index([createdAt])
   @@index([websiteId])
@@ -96,27 +95,20 @@ model EventData {
   id                String    @id() @map("event_id") @db.VarChar(36)
   websiteEventId    String    @map("website_event_id") @db.VarChar(36)
   websiteId         String    @map("website_id") @db.VarChar(36)
-  sessionId         String    @map("session_id") @db.VarChar(36)
-  urlPath           String    @map("url_path") @db.VarChar(500)
-  eventName         String    @map("event_name") @db.VarChar(500)
   eventKey          String    @map("event_key") @db.VarChar(500)
-  eventStringValue  String    @map("event_string_value") @db.VarChar(500)
+  eventStringValue  String?   @map("event_string_value") @db.VarChar(500)
-  eventNumericValue Decimal   @map("event_numeric_value") @db.Decimal(19,4)
+  eventNumericValue Decimal?  @map("event_numeric_value") @db.Decimal(19, 4)
   eventDateValue    DateTime? @map("event_date_value") @db.Timestamp(0)
   eventDataType     Int       @map("event_data_type") @db.UnsignedInt
-  createdAt         DateTime? @default(now()) @map("created_at")@db.Timestamp(0)
+  createdAt         DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
 
   website      Website      @relation(fields: [websiteId], references: [id])
   websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
-  session      Session      @relation(fields: [sessionId], references: [id])
 
   @@index([createdAt])
-  @@index([sessionId])
   @@index([websiteId])
   @@index([websiteEventId])
   @@index([websiteId, websiteEventId, createdAt])
-  @@index([websiteId, sessionId, createdAt])
-  @@index([websiteId, sessionId, websiteEventId, createdAt])
   @@map("event_data")
 }
 

db/postgresql/migrations/09_add_new_event_data/migration.sql

@@ -1,38 +0,0 @@
--- CreateTable
-CREATE TABLE "event_data" (
-    "event_id" UUID NOT NULL,
-    "website_event_id" UUID NOT NULL,
-    "website_id" UUID NOT NULL,
-    "session_id" UUID NOT NULL,
-    "url_path" VARCHAR(500) NOT NULL,
-    "event_name" VARCHAR(500) NOT NULL,
-    "event_key" VARCHAR(500) NOT NULL,
-    "event_string_value" VARCHAR(500) NOT NULL,
-    "event_numeric_value" DECIMAL(19,4) NOT NULL,
-    "event_date_value" TIMESTAMPTZ(6),
-    "event_data_type" INTEGER NOT NULL,
-    "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
-
-    CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id")
-);
-
--- CreateIndex
-CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at");
-
--- CreateIndex
-CREATE INDEX "event_data_session_id_idx" ON "event_data"("session_id");
-
--- CreateIndex
-CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id");
-
--- CreateIndex
-CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id");
-
--- CreateIndex
-CREATE INDEX "event_data_website_id_website_event_id_created_at_idx" ON "event_data"("website_id", "website_event_id", "created_at");
-
--- CreateIndex
-CREATE INDEX "event_data_website_id_session_id_created_at_idx" ON "event_data"("website_id", "session_id", "created_at");
-
--- CreateIndex
-CREATE INDEX "event_data_website_id_session_id_website_event_id_created_a_idx" ON "event_data"("website_id", "session_id", "website_event_id", "created_at");

db/postgresql/migrations/09_event_data/migration.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "event_data" (
+    "event_id" UUID NOT NULL,
+    "website_id" UUID NOT NULL,
+    "website_event_id" UUID NOT NULL,
+    "event_key" VARCHAR(500) NOT NULL,
+    "event_string_value" VARCHAR(500),
+    "event_numeric_value" DECIMAL(19,4),
+    "event_date_value" TIMESTAMPTZ(6),
+    "event_data_type" INTEGER NOT NULL,
+    "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "event_data_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateIndex
+CREATE INDEX "event_data_created_at_idx" ON "event_data"("created_at");
+
+-- CreateIndex
+CREATE INDEX "event_data_website_id_idx" ON "event_data"("website_id");
+
+-- CreateIndex
+CREATE INDEX "event_data_website_event_id_idx" ON "event_data"("website_event_id");

db/postgresql/schema.prisma

@@ -39,7 +39,6 @@ model Session {
   createdAt    DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
 
   websiteEvent WebsiteEvent[]
-  eventData    EventData[]
 
   @@index([createdAt])
   @@index([websiteId])
@@ -59,7 +58,7 @@ model Website {
 
   user        User?         @relation(fields: [userId], references: [id])
   teamWebsite TeamWebsite[]
-  eventData   EventData[]
+  EventData   EventData[]
 
   @@index([userId])
   @@index([createdAt])
@@ -94,29 +93,21 @@ model WebsiteEvent {
 
 model EventData {
   id                String    @id() @map("event_id") @db.Uuid
-  websiteEventId    String    @map("website_event_id") @db.Uuid
   websiteId         String    @map("website_id") @db.Uuid
-  sessionId         String    @map("session_id") @db.Uuid
+  websiteEventId    String    @map("website_event_id") @db.Uuid
-  urlPath           String    @map("url_path") @db.VarChar(500)
-  eventName         String    @map("event_name") @db.VarChar(500)
   eventKey          String    @map("event_key") @db.VarChar(500)
-  eventStringValue  String    @map("event_string_value") @db.VarChar(500)
+  eventStringValue  String?   @map("event_string_value") @db.VarChar(500)
-  eventNumericValue Int       @map("event_numeric_value") @db.Integer
+  eventNumericValue Decimal?  @map("event_numeric_value") @db.Decimal(19, 4)
   eventDateValue    DateTime? @map("event_date_value") @db.Timestamptz(6)
   eventDataType     Int       @map("event_data_type") @db.Integer
   createdAt         DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
 
   website      Website      @relation(fields: [websiteId], references: [id])
   websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
-  session      Session      @relation(fields: [sessionId], references: [id])
 
   @@index([createdAt])
-  @@index([sessionId])
   @@index([websiteId])
   @@index([websiteEventId])
-  @@index([websiteId, websiteEventId, createdAt])
-  @@index([websiteId, sessionId, createdAt])
-  @@index([websiteId, sessionId, websiteEventId, createdAt])
   @@map("event_data")
 }
 
@@ -162,4 +153,4 @@ model TeamWebsite {
   @@index([teamId])
   @@index([websiteId])
   @@map("team_website")
-}
+}

lib/prisma.ts

@@ -72,33 +72,34 @@ function getEventDataFilterQuery(
   }[],
   params: any[],
 ) {
-  const query = filters.reduce((ac, cv, i) => {
+  const query = filters.reduce((ac, cv) => {
-    const type = getEventDataType(cv);
+    const type = getEventDataType(cv.eventValue);
 
     let value = cv.eventValue;
 
+    ac.push(`and (event_key = $${params.length + 1}`);
+    params.push(cv.eventKey);
+
     switch (type) {
       case 'number':
-        ac.push(`and event_numeric_value = {${cv.eventKey}${i}:UInt64}`);
+        ac.push(`and event_numeric_value = $${params.length + 1})`);
-        params.push(cv.eventValue);
+        params.push(value);
         break;
       case 'string':
-        ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
+        ac.push(`and event_string_value = $${params.length + 1})`);
         params.push(decodeURIComponent(cv.eventValue as string));
         break;
       case 'boolean':
-        ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
+        ac.push(`and event_string_value = $${params.length + 1})`);
         params.push(decodeURIComponent(cv.eventValue as string));
         value = cv ? 'true' : 'false';
         break;
       case 'date':
-        ac.push(`and event_date_value = {${cv.eventKey}${i}:DateTime('UTC')}`);
+        ac.push(`and event_date_value = $${params.length + 1})`);
         params.push(cv.eventValue);
         break;
     }
 
-    params[`${cv.eventKey}${i}`] = value;
-
     return ac;
   }, []);
 

pages/api/send.ts

@@ -99,8 +99,8 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
     referrerDomain = newRef.hostname;
     referrerQuery = newRef.search.substring(1);
   } catch {
-    referrerPath = referrer.split('?')[0];
+    referrerPath = referrer?.split('?')[0];
-    referrerQuery = referrer.split('?')[1];
+    referrerQuery = referrer?.split('?')[1];
   }
 
   if (process.env.REMOVE_TRAILING_SLASH) {

queries/analytics/event/saveEvent.ts

@@ -40,11 +40,13 @@ async function relationalQuery(data: {
   eventName?: string;
   eventData?: any;
 }) {
-  const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data;
+  const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data;
+  const website = await cache.fetchWebsite(websiteId);
+  const websiteEventId = uuid();
 
-  return prisma.client.websiteEvent.create({
+  const websiteEvent = prisma.client.websiteEvent.create({
     data: {
-      id: uuid(),
+      id: websiteEventId,
       websiteId,
       sessionId,
       urlPath: urlPath?.substring(0, URL_LENGTH),
@@ -54,6 +56,20 @@ async function relationalQuery(data: {
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
     },
   });
+
+  if (eventData) {
+    await saveEventData({
+      websiteId,
+      sessionId,
+      eventId: websiteEventId,
+      revId: website?.revId,
+      urlPath: urlPath?.substring(0, URL_LENGTH),
+      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      eventData,
+    });
+  }
+
+  return websiteEvent;
 }
 
 async function clickhouseQuery(data: {

queries/analytics/eventData/getEventData.ts

@@ -48,7 +48,7 @@ async function relationalQuery(
 ) {
   const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
   const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
-  const params: any = [websiteId, startDate, endDate, eventName];
+  const params: any = [websiteId, startDate, endDate, eventName || ''];
 
   return rawQuery(
     `select
@@ -59,6 +59,11 @@ async function relationalQuery(
           timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
         }
     from event_data
+      ${
+        eventName || urlPath
+          ? 'join website_event on event_data.id = website_event.website_event_id'
+          : ''
+      }
     where website_id = $1${toUuid()}
       and created_at between $2 and $3
       ${eventName ? `and eventName = $4` : ''}

queries/analytics/eventData/saveEventData.ts

@@ -1,18 +1,21 @@
+import { Prisma } from '@prisma/client';
 import { EVENT_DATA_TYPE } from 'lib/constants';
+import { uuid } from 'lib/crypto';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import { flattenJSON } from 'lib/eventData';
 import kafka from 'lib/kafka';
+import prisma from 'lib/prisma';
 import { EventData } from 'lib/types';
 
 export async function saveEventData(args: {
   websiteId: string;
-  sessionId: string;
   eventId: string;
-  revId: number;
+  sessionId?: string;
-  urlPath: string;
+  revId?: number;
-  eventName: string;
+  urlPath?: string;
+  eventName?: string;
   eventData: EventData;
-  createdAt: string;
+  createdAt?: string;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(args),
@@ -22,25 +25,44 @@ export async function saveEventData(args: {
 
 async function relationalQuery(data: {
   websiteId: string;
-  sessionId: string;
   eventId: string;
-  revId: number;
-  eventName: string;
   eventData: EventData;
-  createdAt: string;
+}): Promise<Prisma.BatchPayload> {
-}) {
+  const { websiteId, eventId, eventData } = data;
-  return data;
+
+  const jsonKeys = flattenJSON(eventData);
+
+  //id, websiteEventId, eventStringValue
+  const flattendData = jsonKeys.map(a => ({
+    id: uuid(),
+    websiteEventId: eventId,
+    websiteId,
+    eventKey: a.key,
+    eventStringValue:
+      a.eventDataType === EVENT_DATA_TYPE.string ||
+      a.eventDataType === EVENT_DATA_TYPE.boolean ||
+      a.eventDataType === EVENT_DATA_TYPE.array
+        ? a.value
+        : null,
+    eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
+    eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,
+    eventDataType: a.eventDataType,
+  }));
+
+  return prisma.client.eventData.createMany({
+    data: flattendData,
+  });
 }
 
 async function clickhouseQuery(data: {
   websiteId: string;
-  sessionId: string;
   eventId: string;
-  revId: number;
+  sessionId?: string;
-  urlPath: string;
+  revId?: number;
-  eventName: string;
+  urlPath?: string;
+  eventName?: string;
   eventData: EventData;
-  createdAt: string;
+  createdAt?: string;
 }) {
   const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data;
 
@@ -67,7 +89,6 @@ async function clickhouseQuery(data: {
     event_data_type: a.eventDataType,
     created_at: createdAt,
   }));
-  ``;
 
   await sendMessages(messages, 'event_data');
 

queries/index.js

@@ -4,7 +4,7 @@ export * from './admin/user';
 export * from './admin/website';
 export * from './analytics/event/getEventMetrics';
 export * from './analytics/event/getEvents';
-export * from './analytics/event/getEventData';
+export * from './analytics/eventData/getEventData';
 export * from './analytics/event/saveEvent';
 export * from './analytics/pageview/getPageviewMetrics';
 export * from './analytics/pageview/getPageviews';