ambiser
/
umami
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add eventData back.

pull/1841/head
Brian Cao 2023-03-23 00:36:38 -07:00
parent 30d2163610
commit 0d3c159084
9 changed files with 313 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db/clickhouse/schema.sql
View File

@ -106,13 +106,13 @@ CREATE TABLE event_data
event_name String,
event_key String,
event_string_value Nullable(String),
event_numeric_value Nullable(UInt32),
event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
event_date_value Nullable(DateTime('UTC')),
event_data_type UInt32,
created_at DateTime('UTC')
)
engine = MergeTree
ORDER BY (website_id, session_id, event_id, event_key, created_at)
ORDER BY (website_id, event_id, event_key, created_at)
SETTINGS index_granularity = 8192;
CREATE TABLE event_data_queue (
@ -124,7 +124,7 @@ CREATE TABLE event_data_queue (
event_name String,
event_key String,
event_string_value Nullable(String),
event_numeric_value Nullable(UInt64),
event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625
event_date_value Nullable(DateTime('UTC')),
event_data_type UInt32,
created_at DateTime('UTC')

37
db/postgresql/schema.prisma
View File

@ -17,7 +17,7 @@ model User {
updatedAt DateTime? @map("updated_at") @db.Timestamptz(6)
deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6)
Website Website[]
website Website[]
teamUser TeamUser[]
@@map("user")
@ -38,6 +38,9 @@ model Session {
city String? @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
websiteEvent WebsiteEvent[]
eventData EventData[]
@@index([createdAt])
@@index([websiteId])
@@map("session")
@ -56,6 +59,7 @@ model Website {
user User? @relation(fields: [userId], references: [id])
teamWebsite TeamWebsite[]
eventData EventData[]
@@index([userId])
@@index([createdAt])
@ -77,6 +81,9 @@ model WebsiteEvent {
eventType Int @default(1) @map("event_type") @db.Integer
eventName String? @map("event_name") @db.VarChar(50)
eventData EventData[]
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([sessionId])
@@index([websiteId])
@ -85,6 +92,34 @@ model WebsiteEvent {
@@map("website_event")
}
model EventData {
id String @id() @map("event_id") @db.Uuid
websiteEventId String @map("website_event_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
urlPath String @map("url_path") @db.VarChar(500)
eventName String @map("event_name") @db.VarChar(500)
eventKey String @map("event_key") @db.VarChar(500)
eventStringValue String @map("event_string_value") @db.VarChar(500)
eventNumericValue Int @map("event_numeric_value") @db.Integer
eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6)
eventDataType Int @map("event_data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([createdAt])
@@index([sessionId])
@@index([websiteId])
@@index([websiteEventId])
@@index([websiteId, websiteEventId, createdAt])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, sessionId, websiteEventId, createdAt])
@@map("event_data")
}
model Team {
id String @id() @unique() @map("team_id") @db.Uuid
name String @db.VarChar(50)

41
lib/clickhouse.ts
View File

@ -3,6 +3,7 @@ import dateFormat from 'dateformat';
import debug from 'debug';
import { FILTER_IGNORED } from 'lib/constants';
import { CLICKHOUSE } from 'lib/db';
import { getEventDataType } from './eventData';
export const CLICKHOUSE_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%M:00',
@ -64,6 +65,45 @@ function getBetweenDates(field, startAt, endAt) {
return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`;
}
function getEventDataFilterQuery(
filters: {
eventKey?: string;
eventValue?: string | number | boolean | Date;
}[] = [],
params: any,
) {
const query = filters.reduce((ac, cv, i) => {
const type = getEventDataType(cv.eventValue);
let value = cv.eventValue;
ac.push(`and (event_key = {eventKey${i}:String}`);
switch (type) {
case 'number':
ac.push(`and event_numeric_value = {eventValue${i}:UInt64})`);
break;
case 'string':
ac.push(`and event_string_value = {eventValue${i}:String})`);
break;
case 'boolean':
ac.push(`and event_string_value = {eventValue${i}:String})`);
value = cv ? 'true' : 'false';
break;
case 'date':
ac.push(`and event_date_value = {eventValue${i}:DateTime('UTC')})`);
break;
}
params[`eventKey${i}`] = cv.eventKey;
params[`eventValue${i}`] = value;
return ac;
}, []);
return query.join('\n');
}
function getFilterQuery(filters = {}, params = {}) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
@ -189,6 +229,7 @@ export default {
getDateFormat,
getBetweenDates,
getFilterQuery,
getEventDataFilterQuery,
parseFilters,
findUnique,
findFirst,

12
lib/eventData.ts
View File

@ -25,8 +25,18 @@ export function flattenJSON(
).keyValues;
}
export function getEventDataType(value: any): string {
let type: string = typeof value;
if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
type = 'date';
}
return type;
}
function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
const type = isValid(value) || isValid(parseISO(value)) ? 'date' : typeof value;
const type = getEventDataType(value);
let eventDataType = null;

42
lib/prisma.ts
View File

@ -2,6 +2,7 @@ import prisma from '@umami/prisma-client';
import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { FILTER_IGNORED } from 'lib/constants';
import { getEventDataType } from './eventData';
const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00',
@ -64,6 +65,46 @@ function getTimestampInterval(field: string): string {
}
}
function getEventDataFilterQuery(
filters: {
eventKey?: string;
eventValue?: string | number | boolean | Date;
}[],
params: any[],
) {
const query = filters.reduce((ac, cv, i) => {
const type = getEventDataType(cv);
let value = cv.eventValue;
switch (type) {
case 'number':
ac.push(`and event_numeric_value = {${cv.eventKey}${i}:UInt64}`);
params.push(cv.eventValue);
break;
case 'string':
ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
params.push(decodeURIComponent(cv.eventValue as string));
break;
case 'boolean':
ac.push(`and event_string_value = {${cv.eventKey}${i}:String}`);
params.push(decodeURIComponent(cv.eventValue as string));
value = cv ? 'true' : 'false';
break;
case 'date':
ac.push(`and event_date_value = {${cv.eventKey}${i}:DateTime('UTC')}`);
params.push(cv.eventValue);
break;
}
params[`${cv.eventKey}${i}`] = value;
return ac;
}, []);
return query.join('\n');
}
function getFilterQuery(filters = {}, params = []): string {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
@ -173,6 +214,7 @@ export default {
getDateQuery,
getTimestampInterval,
getFilterQuery,
getEventDataFilterQuery,
toUuid,
parseFilters,
rawQuery,

7
lib/types.ts
View File

@ -75,6 +75,13 @@ export interface WebsiteEventMetric {
y: number;
}
export interface WebsiteEventDataMetric {
x: string;
t: string;
eventName?: string;
urlPath?: string;
}
export interface WebsitePageviews {
pageviews: {
t: string;

60
pages/api/websites/[id]/eventData.ts Normal file
View File

@ -0,0 +1,60 @@
import { canViewWebsite } from 'lib/auth';
import { useAuth, useCors } from 'lib/middleware';
import { NextApiRequestQueryBody, WebsiteEventDataMetric } from 'lib/types';
import { NextApiResponse } from 'next';
import { methodNotAllowed, ok, unauthorized } from 'next-basics';
import { getEventData } from 'queries';
export interface WebsiteEventDataRequestQuery {
id: string;
}
export interface WebsiteEventDataRequestBody {
startAt: string;
endAt: string;
eventName?: string;
urlPath?: string;
timeSeries?: {
unit: string;
timezone: string;
};
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
}
export default async (
req: NextApiRequestQueryBody<WebsiteEventDataRequestQuery, WebsiteEventDataRequestBody>,
res: NextApiResponse<WebsiteEventDataMetric[]>,
) => {
await useCors(req, res);
await useAuth(req, res);
const { id: websiteId } = req.query;
if (req.method === 'POST') {
if (!(await canViewWebsite(req.auth, websiteId))) {
return unauthorized(res);
}
const { startAt, endAt, eventName, urlPath, filters } = req.body;
const startDate = new Date(+startAt);
const endDate = new Date(+endAt);
const events = await getEventData(websiteId, {
startDate,
endDate,
eventName,
urlPath,
filters,
});
return ok(res, events);
}
return methodNotAllowed(res);
};

112
queries/analytics/event/getEventData.ts Normal file
View File

@ -0,0 +1,112 @@
import cache from 'lib/cache';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { WebsiteEventDataMetric } from 'lib/types';
export async function getEventData(
...args: [
websiteId: string,
data: {
startDate: Date;
endDate: Date;
eventName: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
},
]
): Promise<WebsiteEventDataMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
data: {
startDate: Date;
endDate: Date;
timeSeries?: {
unit: string;
timezone: string;
};
eventName: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
},
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
const params: any = [websiteId, startDate, endDate, eventName];
return rawQuery(
`select
count(*) x
${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''}
${
timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
}
from event_data
where website_id = $1${toUuid()}
and created_at between $2 and $3
${eventName ? `and eventName = $4` : ''}
${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
data: {
startDate: Date;
endDate: Date;
timeSeries?: {
unit: string;
timezone: string;
};
eventName?: string;
urlPath?: string;
filters: [
{
eventKey?: string;
eventValue?: string | number | boolean | Date;
},
];
},
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { rawQuery, getBetweenDates, getDateQuery, getEventDataFilterQuery } = clickhouse;
const website = await cache.fetchWebsite(websiteId);
const params = { websiteId, revId: website?.revId || 0 };
return rawQuery(
`select
count(*) x
${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''}
${
timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
}
from event_data
where website_id = {websiteId:UUID}
and rev_id = {revId:UInt32}
${eventName ? `and eventName = ${eventName}` : ''}
and ${getBetweenDates('created_at', startDate, endDate)}
${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`,
params,
);
}

1
queries/index.js
View File

@ -4,6 +4,7 @@ export * from './admin/user';
export * from './admin/website';
export * from './analytics/event/getEventMetrics';
export * from './analytics/event/getEvents';
export * from './analytics/event/getEventData';
export * from './analytics/event/saveEvent';
export * from './analytics/pageview/getPageviewMetrics';
export * from './analytics/pageview/getPageviews';