diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index de9c109f..525b20db 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -106,13 +106,13 @@ CREATE TABLE event_data event_name String, event_key String, event_string_value Nullable(String), - event_numeric_value Nullable(UInt32), + event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 event_date_value Nullable(DateTime('UTC')), event_data_type UInt32, created_at DateTime('UTC') ) engine = MergeTree - ORDER BY (website_id, session_id, event_id, event_key, created_at) + ORDER BY (website_id, event_id, event_key, created_at) SETTINGS index_granularity = 8192; CREATE TABLE event_data_queue ( @@ -124,7 +124,7 @@ CREATE TABLE event_data_queue ( event_name String, event_key String, event_string_value Nullable(String), - event_numeric_value Nullable(UInt64), + event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 event_date_value Nullable(DateTime('UTC')), event_data_type UInt32, created_at DateTime('UTC') diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 82b22c64..7033a25f 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -17,7 +17,7 @@ model User { updatedAt DateTime? @map("updated_at") @db.Timestamptz(6) deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6) - Website Website[] + website Website[] teamUser TeamUser[] @@map("user") @@ -38,6 +38,9 @@ model Session { city String? @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + websiteEvent WebsiteEvent[] + eventData EventData[] + @@index([createdAt]) @@index([websiteId]) @@map("session") @@ -56,6 +59,7 @@ model Website { user User? @relation(fields: [userId], references: [id]) teamWebsite TeamWebsite[] + eventData EventData[] @@index([userId]) @@index([createdAt]) @@ -77,6 +81,9 @@ model WebsiteEvent { eventType Int @default(1) @map("event_type") @db.Integer eventName String? @map("event_name") @db.VarChar(50) + eventData EventData[] + session Session @relation(fields: [sessionId], references: [id]) + @@index([createdAt]) @@index([sessionId]) @@index([websiteId]) @@ -85,6 +92,34 @@ model WebsiteEvent { @@map("website_event") } +model EventData { + id String @id() @map("event_id") @db.Uuid + websiteEventId String @map("website_event_id") @db.Uuid + websiteId String @map("website_id") @db.Uuid + sessionId String @map("session_id") @db.Uuid + urlPath String @map("url_path") @db.VarChar(500) + eventName String @map("event_name") @db.VarChar(500) + eventKey String @map("event_key") @db.VarChar(500) + eventStringValue String @map("event_string_value") @db.VarChar(500) + eventNumericValue Int @map("event_numeric_value") @db.Integer + eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6) + eventDataType Int @map("event_data_type") @db.Integer + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + + website Website @relation(fields: [websiteId], references: [id]) + websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) + session Session @relation(fields: [sessionId], references: [id]) + + @@index([createdAt]) + @@index([sessionId]) + @@index([websiteId]) + @@index([websiteEventId]) + @@index([websiteId, websiteEventId, createdAt]) + @@index([websiteId, sessionId, createdAt]) + @@index([websiteId, sessionId, websiteEventId, createdAt]) + @@map("event_data") +} + model Team { id String @id() @unique() @map("team_id") @db.Uuid name String @db.VarChar(50) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 073a92e7..d9e6a02b 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -3,6 +3,7 @@ import dateFormat from 'dateformat'; import debug from 'debug'; import { FILTER_IGNORED } from 'lib/constants'; import { CLICKHOUSE } from 'lib/db'; +import { getEventDataType } from './eventData'; export const CLICKHOUSE_DATE_FORMATS = { minute: '%Y-%m-%d %H:%M:00', @@ -64,6 +65,45 @@ function getBetweenDates(field, startAt, endAt) { return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`; } +function getEventDataFilterQuery( + filters: { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }[] = [], + params: any, +) { + const query = filters.reduce((ac, cv, i) => { + const type = getEventDataType(cv.eventValue); + + let value = cv.eventValue; + + ac.push(`and (event_key = {eventKey${i}:String}`); + + switch (type) { + case 'number': + ac.push(`and event_numeric_value = {eventValue${i}:UInt64})`); + break; + case 'string': + ac.push(`and event_string_value = {eventValue${i}:String})`); + break; + case 'boolean': + ac.push(`and event_string_value = {eventValue${i}:String})`); + value = cv ? 'true' : 'false'; + break; + case 'date': + ac.push(`and event_date_value = {eventValue${i}:DateTime('UTC')})`); + break; + } + + params[`eventKey${i}`] = cv.eventKey; + params[`eventValue${i}`] = value; + + return ac; + }, []); + + return query.join('\n'); +} + function getFilterQuery(filters = {}, params = {}) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -189,6 +229,7 @@ export default { getDateFormat, getBetweenDates, getFilterQuery, + getEventDataFilterQuery, parseFilters, findUnique, findFirst, diff --git a/lib/eventData.ts b/lib/eventData.ts index 31cd5695..4588d081 100644 --- a/lib/eventData.ts +++ b/lib/eventData.ts @@ -25,8 +25,18 @@ export function flattenJSON( ).keyValues; } +export function getEventDataType(value: any): string { + let type: string = typeof value; + + if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) { + type = 'date'; + } + + return type; +} + function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) { - const type = isValid(value) || isValid(parseISO(value)) ? 'date' : typeof value; + const type = getEventDataType(value); let eventDataType = null; diff --git a/lib/prisma.ts b/lib/prisma.ts index 20a5e4a6..ec7c6527 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -2,6 +2,7 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; import { FILTER_IGNORED } from 'lib/constants'; +import { getEventDataType } from './eventData'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -64,6 +65,46 @@ function getTimestampInterval(field: string): string { } } +function getEventDataFilterQuery( + filters: { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }[], + params: any[], +) { + const query = filters.reduce((ac, cv, i) => { + const type = getEventDataType(cv); + + let value = cv.eventValue; + + switch (type) { + case 'number': + ac.push(`and event_numeric_value = {${cv.eventKey}${i}:UInt64}`); + params.push(cv.eventValue); + break; + case 'string': + ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`); + params.push(decodeURIComponent(cv.eventValue as string)); + break; + case 'boolean': + ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`); + params.push(decodeURIComponent(cv.eventValue as string)); + value = cv ? 'true' : 'false'; + break; + case 'date': + ac.push(`and event_date_value = {${cv.eventKey}${i}:DateTime('UTC')}`); + params.push(cv.eventValue); + break; + } + + params[`${cv.eventKey}${i}`] = value; + + return ac; + }, []); + + return query.join('\n'); +} + function getFilterQuery(filters = {}, params = []): string { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -173,6 +214,7 @@ export default { getDateQuery, getTimestampInterval, getFilterQuery, + getEventDataFilterQuery, toUuid, parseFilters, rawQuery, diff --git a/lib/types.ts b/lib/types.ts index 20d49610..c818d6ff 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -75,6 +75,13 @@ export interface WebsiteEventMetric { y: number; } +export interface WebsiteEventDataMetric { + x: string; + t: string; + eventName?: string; + urlPath?: string; +} + export interface WebsitePageviews { pageviews: { t: string; diff --git a/pages/api/websites/[id]/eventData.ts b/pages/api/websites/[id]/eventData.ts new file mode 100644 index 00000000..65c4d687 --- /dev/null +++ b/pages/api/websites/[id]/eventData.ts @@ -0,0 +1,60 @@ +import { canViewWebsite } from 'lib/auth'; +import { useAuth, useCors } from 'lib/middleware'; +import { NextApiRequestQueryBody, WebsiteEventDataMetric } from 'lib/types'; +import { NextApiResponse } from 'next'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { getEventData } from 'queries'; + +export interface WebsiteEventDataRequestQuery { + id: string; +} + +export interface WebsiteEventDataRequestBody { + startAt: string; + endAt: string; + eventName?: string; + urlPath?: string; + timeSeries?: { + unit: string; + timezone: string; + }; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; +} + +export default async ( + req: NextApiRequestQueryBody, + res: NextApiResponse, +) => { + await useCors(req, res); + await useAuth(req, res); + + const { id: websiteId } = req.query; + + if (req.method === 'POST') { + if (!(await canViewWebsite(req.auth, websiteId))) { + return unauthorized(res); + } + + const { startAt, endAt, eventName, urlPath, filters } = req.body; + + const startDate = new Date(+startAt); + const endDate = new Date(+endAt); + + const events = await getEventData(websiteId, { + startDate, + endDate, + eventName, + urlPath, + filters, + }); + + return ok(res, events); + } + + return methodNotAllowed(res); +}; diff --git a/queries/analytics/event/getEventData.ts b/queries/analytics/event/getEventData.ts new file mode 100644 index 00000000..bae64db2 --- /dev/null +++ b/queries/analytics/event/getEventData.ts @@ -0,0 +1,112 @@ +import cache from 'lib/cache'; +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; +import { WebsiteEventDataMetric } from 'lib/types'; + +export async function getEventData( + ...args: [ + websiteId: string, + data: { + startDate: Date; + endDate: Date; + eventName: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, + ] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + data: { + startDate: Date; + endDate: Date; + timeSeries?: { + unit: string; + timezone: string; + }; + eventName: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, +) { + const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; + const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; + const params: any = [websiteId, startDate, endDate, eventName]; + + return rawQuery( + `select + count(*) x + ${eventName ? `,event_name eventName` : ''} + ${urlPath ? `,url_path urlPath` : ''} + ${ + timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : '' + } + from event_data + where website_id = $1${toUuid()} + and created_at between $2 and $3 + ${eventName ? `and eventName = $4` : ''} + ${getEventDataFilterQuery(filters, params)} + ${timeSeries ? 'group by t' : ''}`, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + data: { + startDate: Date; + endDate: Date; + timeSeries?: { + unit: string; + timezone: string; + }; + eventName?: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, +) { + const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; + const { rawQuery, getBetweenDates, getDateQuery, getEventDataFilterQuery } = clickhouse; + const website = await cache.fetchWebsite(websiteId); + const params = { websiteId, revId: website?.revId || 0 }; + + return rawQuery( + `select + count(*) x + ${eventName ? `,event_name eventName` : ''} + ${urlPath ? `,url_path urlPath` : ''} + ${ + timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : '' + } + from event_data + where website_id = {websiteId:UUID} + and rev_id = {revId:UInt32} + ${eventName ? `and eventName = ${eventName}` : ''} + and ${getBetweenDates('created_at', startDate, endDate)} + ${getEventDataFilterQuery(filters, params)} + ${timeSeries ? 'group by t' : ''}`, + params, + ); +} diff --git a/queries/index.js b/queries/index.js index e59413b9..2d1931ee 100644 --- a/queries/index.js +++ b/queries/index.js @@ -4,6 +4,7 @@ export * from './admin/user'; export * from './admin/website'; export * from './analytics/event/getEventMetrics'; export * from './analytics/event/getEvents'; +export * from './analytics/event/getEventData'; export * from './analytics/event/saveEvent'; export * from './analytics/pageview/getPageviewMetrics'; export * from './analytics/pageview/getPageviews';