From bb30b43a8eb0201761a907df10078feac592b210 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 15 May 2023 15:49:21 +0900 Subject: [PATCH 1/7] feat: support mongodb connection --- db/mongodb/schema.prisma | 155 ++++++++++++++++++ lib/db.js | 5 +- lib/prisma.ts | 7 +- queries/admin/user.ts | 23 ++- .../analytics/pageview/getPageviewStats.ts | 31 +++- queries/analytics/stats/getActiveVisitors.ts | 62 +++++-- queries/analytics/stats/getWebsiteStats.ts | 86 +++++++++- scripts/check-db.js | 27 ++- scripts/copy-db-files.js | 4 +- 9 files changed, 370 insertions(+), 30 deletions(-) create mode 100644 db/mongodb/schema.prisma diff --git a/db/mongodb/schema.prisma b/db/mongodb/schema.prisma new file mode 100644 index 00000000..b6e463bc --- /dev/null +++ b/db/mongodb/schema.prisma @@ -0,0 +1,155 @@ +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "mongodb" + url = env("DATABASE_URL") + relationMode = "prisma" +} + +model User { + id String @id @map("_id") @db.String + username String @unique @db.String + password String @db.String + role String @map("role") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + deletedAt DateTime? @map("deleted_at") @db.Date + + website Website[] + teamUser TeamUser[] + + @@map("user") +} + +model Session { + id String @id @map("_id") @db.String + websiteId String @map("website_id") @db.String + hostname String? @db.String + browser String? @db.String + os String? @db.String + device String? @db.String + screen String? @db.String + language String? @db.String + country String? @db.String + subdivision1 String? @db.String + subdivision2 String? @db.String + city String? @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + websiteEvent WebsiteEvent[] + + @@index([createdAt]) + @@index([websiteId]) + @@map("session") +} + +model Website { + id String @id @map("_id") @db.String + name String @db.String + domain String? @db.String + shareId String? @unique @map("share_id") @db.String + resetAt DateTime? @map("reset_at") @db.Date + userId String? @map("user_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + deletedAt DateTime? @map("deleted_at") @db.Date + + user User? @relation(fields: [userId], references: [id]) + teamWebsite TeamWebsite[] + eventData EventData[] + + @@index([userId]) + @@index([createdAt]) + @@map("website") +} + +model WebsiteEvent { + id String @id() @map("_id") @db.String + websiteId String @map("website_id") @db.String + sessionId String @map("session_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + urlPath String @map("url_path") @db.String + urlQuery String? @map("url_query") @db.String + referrerPath String? @map("referrer_path") @db.String + referrerQuery String? @map("referrer_query") @db.String + referrerDomain String? @map("referrer_domain") @db.String + pageTitle String? @map("page_title") @db.String + eventType Int @default(1) @map("event_type") @db.Int + eventName String? @map("event_name") @db.String + + eventData EventData[] + session Session @relation(fields: [sessionId], references: [id]) + + @@index([createdAt]) + @@index([sessionId]) + @@index([websiteId]) + @@index([websiteId, createdAt]) + @@index([websiteId, sessionId, createdAt]) + @@map("website_event") +} + +model EventData { + id String @id() @map("_id") @db.String + websiteEventId String @map("website_event_id") @db.String + websiteId String @map("website_id") @db.String + eventKey String @map("event_key") @db.String + eventStringValue String? @map("event_string_value") @db.String + eventNumericValue Float? @map("event_numeric_value") @db.Double // (19, 4) + eventDateValue DateTime? @map("event_date_value") @db.Date + eventDataType Int @map("event_data_type") @db.Int + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + website Website @relation(fields: [websiteId], references: [id]) + websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) + + @@index([createdAt]) + @@index([websiteId]) + @@index([websiteEventId]) + @@index([websiteId, websiteEventId, createdAt]) + @@map("event_data") +} + +model Team { + id String @id() @map("_id") @db.String + name String @db.String + accessCode String? @unique @map("access_code") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + + teamUser TeamUser[] + teamWebsite TeamWebsite[] + + @@map("team") +} + +model TeamUser { + id String @id() @map("_id") @db.String + teamId String @map("team_id") @db.String + userId String @map("user_id") @db.String + role String @map("role") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + + team Team @relation(fields: [teamId], references: [id]) + user User @relation(fields: [userId], references: [id]) + + @@index([teamId]) + @@index([userId]) + @@map("team_user") +} + +model TeamWebsite { + id String @id() @map("_id") @db.String + teamId String @map("team_id") @db.String + websiteId String @map("website_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + team Team @relation(fields: [teamId], references: [id]) + website Website @relation(fields: [websiteId], references: [id]) + + @@index([teamId]) + @@index([websiteId]) + @@map("team_website") +} diff --git a/lib/db.js b/lib/db.js index 19e46a3d..d85b8aa7 100644 --- a/lib/db.js +++ b/lib/db.js @@ -4,6 +4,7 @@ export const MYSQL = 'mysql'; export const CLICKHOUSE = 'clickhouse'; export const KAFKA = 'kafka'; export const KAFKA_PRODUCER = 'kafka-producer'; +export const MONGODB = 'mongodb'; // Fixes issue with converting bigint values BigInt.prototype.toJSON = function () { @@ -15,6 +16,8 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return POSTGRESQL; + } else if (type === 'mongodb+srv') { + return MONGODB; } return type; @@ -23,7 +26,7 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { export async function runQuery(queries) { const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL); - if (db === POSTGRESQL || db === MYSQL) { + if (db === POSTGRESQL || db === MYSQL || db === MONGODB) { return queries[PRISMA](); } diff --git a/lib/prisma.ts b/lib/prisma.ts index 0a10d981..cf6ec9fe 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -1,6 +1,6 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; -import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; +import { MYSQL, POSTGRESQL, getDatabaseType, MONGODB } from 'lib/db'; import { getEventDataType } from './eventData'; import { FILTER_COLUMNS } from './constants'; @@ -51,6 +51,10 @@ function getDateQuery(field: string, unit: string, timezone?: string): string { return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`; } + + if (db === MONGODB) { + return MYSQL_DATE_FORMATS[unit]; + } } function getTimestampInterval(field: string): string { @@ -152,6 +156,7 @@ async function rawQuery(query: string, params: never[] = []): Promise { export default { ...prisma, + getDatabaseType: () => getDatabaseType(process.env.DATABASE_URL), getDateQuery, getTimestampInterval, getFilterQuery, diff --git a/queries/admin/user.ts b/queries/admin/user.ts index 412c7785..2feb913d 100644 --- a/queries/admin/user.ts +++ b/queries/admin/user.ts @@ -3,6 +3,16 @@ import cache from 'lib/cache'; import { ROLES } from 'lib/constants'; import prisma from 'lib/prisma'; import { Website, User, Roles } from 'lib/types'; +import { getDatabaseType } from '../../lib/db'; + +function whereDeletedAtNotNull() { + const db = getDatabaseType(process.env.DATABASE_URL); + if (db === 'mongodb') { + return { isSet: false }; + } else { + return null; + } +} export async function getUser( where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput, @@ -11,7 +21,14 @@ export async function getUser( const { includePassword = false, showDeleted = false } = options; return prisma.client.user.findFirst({ - where: { ...where, ...(showDeleted ? {} : { deletedAt: null }) }, + where: { + ...where, + ...(showDeleted + ? {} + : { + deletedAt: whereDeletedAtNotNull(), + }), + }, select: { id: true, username: true, @@ -26,7 +43,7 @@ export async function getUsers(): Promise { return prisma.client.user.findMany({ take: 100, where: { - deletedAt: null, + deletedAt: whereDeletedAtNotNull(), }, orderBy: [ { @@ -76,7 +93,7 @@ export async function getUserWebsites(userId: string): Promise { return prisma.client.website.findMany({ where: { userId, - deletedAt: null, + deletedAt: whereDeletedAtNotNull(), }, orderBy: [ { diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index 01e4ab14..9971fc37 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -45,14 +45,34 @@ async function relationalQuery( filters = {}, sessionKey = 'session_id', } = criteria; - const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma; + const { getDatabaseType, toUuid, getDateQuery, parseFilters, rawQuery, client } = prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select ${getDateQuery('website_event.created_at', unit, timezone)} x, + //TODO: 구현해야 함 + + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $project: { + x: { + $dateToString: { + date: '$created_at', + timezone: timezone, + format: getDateQuery('website_event.created_at', unit, timezone), + }, + }, + }, + }, + ], + }); + } else { + return rawQuery( + `select ${getDateQuery('website_event.created_at', unit, timezone)} x, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from website_event ${joinSession} @@ -62,8 +82,9 @@ async function relationalQuery( and event_type = ${EVENT_TYPE.pageView} ${filterQuery} group by 1`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/queries/analytics/stats/getActiveVisitors.ts b/queries/analytics/stats/getActiveVisitors.ts index 89f092c1..5d38d81f 100644 --- a/queries/analytics/stats/getActiveVisitors.ts +++ b/queries/analytics/stats/getActiveVisitors.ts @@ -11,20 +11,62 @@ export async function getActiveVisitors(...args: [websiteId: string]) { } async function relationalQuery(websiteId: string) { - const { toUuid, rawQuery } = prisma; + const { getDatabaseType, toUuid, rawQuery, client } = prisma; + const db = getDatabaseType(); const date = subMinutes(new Date(), 5); const params: any = [websiteId, date]; - return rawQuery( - `select count(distinct session_id) x - from website_event - join website - on website_event.website_id = website.website_id - where website.website_id = $1${toUuid()} - and website_event.created_at >= $2`, - params, - ); + if (db === 'mongodb') { + const result: any = await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: date.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $group: { + _id: '$session_id', + }, + }, + { + $count: 'x', + }, + ], + }); + if (result.length > 0) { + return { x: result[0].x }; + } else { + return { x: 0 }; + } + } else { + return rawQuery( + `select count(distinct session_id) x + from website_event + join website + on website_event.website_id = website.website_id + where website.website_id = $1${toUuid()} + and website_event.created_at >= $2`, + params, + ); + } } async function clickhouseQuery(websiteId: string) { diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 0021e793..4b8ce5ec 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -21,14 +21,89 @@ async function relationalQuery( criteria: { startDate: Date; endDate: Date; filters: object }, ) { const { startDate, endDate, filters = {} } = criteria; - const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; + const { + getDatabaseType, + toUuid, + getDateQuery, + getTimestampInterval, + parseFilters, + rawQuery, + client, + } = prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select sum(t.c) as "pageviews", + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $project: { + session_id: '$session_id', + hour: { + $toString: { $hour: '$created_at' }, + }, + created_at: '$created_at', + }, + }, + { + $group: { + _id: { + $concat: ['$session_id', ':', '$hour'], + }, + session_id: { $first: '$session_id' }, + hour: { $first: '$hour' }, + count: { $sum: 1 }, + timeMax: { $max: '$created_at' }, + timeMin: { $min: '$created_at' }, + }, + }, + { + $project: { + _id: '$_id', + session_id: '$session_id', + hour: '$hour', + count: '$count', + time: { + $dateDiff: { + endDate: '$timeMax', + startDate: '$timeMin', + unit: 'second', + }, + }, + bounce: { + $cond: { + if: { $eq: ['$count', 1] }, + then: 1, + else: 0, + }, + }, + }, + }, + { + $group: { + _id: '$session_id', + pageviews: { $sum: '$count' }, + bounces: { $sum: '$bounce' }, + totaltime: { $sum: '$time' }, + }, + }, + { + $group: { + _id: '', + pageviews: { $sum: '$pageviews' }, + uniques: { $sum: 1 }, + bounces: { $sum: '$bounces' }, + totaltime: { $sum: '$totaltime' }, + }, + }, + ], + }); + } else { + return rawQuery( + `select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(t.time) as "totaltime" @@ -48,8 +123,9 @@ async function relationalQuery( ${filterQuery} group by 1, 2 ) t`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/scripts/check-db.js b/scripts/check-db.js index 3fd3a908..acf912bf 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -15,6 +15,8 @@ function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return 'postgresql'; + } else if (type === 'mongodb+srv') { + return 'mongodb'; } return type; @@ -50,10 +52,19 @@ async function checkConnection() { } async function checkDatabaseVersion(databaseType) { - const query = await prisma.$queryRaw`select version() as version`; - const version = semver.valid(semver.coerce(query[0].version)); + let version; + if (databaseType === 'mongodb') { + const query = await prisma.$runCommandRaw({ + serverStatus: 1, + }); + version = semver.valid(query.version); + } else { + const query = await prisma.$queryRaw`select version() as version`; + version = semver.valid(semver.coerce(query[0].version)); + } - const minVersion = databaseType === 'postgresql' ? '9.4.0' : '5.7.0'; + const minVersion = + databaseType === 'postgresql' ? '9.4.0' : databaseType === 'mongodb' ? '3.0.0' : '5.7.0'; if (semver.lt(version, minVersion)) { throw new Error( @@ -65,6 +76,10 @@ async function checkDatabaseVersion(databaseType) { } async function checkV1Tables() { + if (databaseType === 'mongodb') { + // Ignore + } + try { await prisma.$queryRaw`select * from account limit 1`; @@ -78,7 +93,11 @@ async function checkV1Tables() { } async function applyMigration() { - console.log(execSync('prisma migrate deploy').toString()); + if (databaseType === 'mongodb') { + console.log(execSync('prisma db push').toString()); + } else { + console.log(execSync('prisma migrate deploy').toString()); + } success('Database is up to date.'); } diff --git a/scripts/copy-db-files.js b/scripts/copy-db-files.js index 15c34674..9bdd8506 100644 --- a/scripts/copy-db-files.js +++ b/scripts/copy-db-files.js @@ -9,6 +9,8 @@ function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return 'postgresql'; + } else if (type === 'mongodb+srv') { + return 'mongodb'; } return type; @@ -16,7 +18,7 @@ function getDatabaseType(url = process.env.DATABASE_URL) { const databaseType = getDatabaseType(); -if (!databaseType || !['mysql', 'postgresql'].includes(databaseType)) { +if (!databaseType || !['mysql', 'postgresql', 'mongodb'].includes(databaseType)) { throw new Error('Missing or invalid database'); } From d5fac29dacf442ccec3dc1c4c45140202efa13f7 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 15 May 2023 16:00:56 +0900 Subject: [PATCH 2/7] ignore checkV1Tables on mongodb --- scripts/check-db.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check-db.js b/scripts/check-db.js index acf912bf..a01c5180 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -78,6 +78,7 @@ async function checkDatabaseVersion(databaseType) { async function checkV1Tables() { if (databaseType === 'mongodb') { // Ignore + return; } try { From ed29fc08e8547d76ff8f17f346d7bed500955c4a Mon Sep 17 00:00:00 2001 From: minkik Date: Thu, 18 May 2023 09:10:35 +0900 Subject: [PATCH 3/7] feat: sql to mongo pipeline --- pages/api/websites/[id]/stats.ts | 24 ++-- queries/analytics/event/getEventMetrics.ts | 104 ++++++++++++++- queries/analytics/eventData/getEventData.ts | 121 +++++++++++++++++- .../analytics/pageview/getPageviewMetrics.ts | 95 +++++++++++++- .../analytics/pageview/getPageviewStats.ts | 100 ++++++++++++++- .../analytics/session/getSessionMetrics.ts | 103 ++++++++++++++- queries/analytics/stats/getWebsiteStats.ts | 44 +++++++ 7 files changed, 558 insertions(+), 33 deletions(-) diff --git a/pages/api/websites/[id]/stats.ts b/pages/api/websites/[id]/stats.ts index 1e2f2292..a61ba302 100644 --- a/pages/api/websites/[id]/stats.ts +++ b/pages/api/websites/[id]/stats.ts @@ -92,14 +92,22 @@ export default async ( city, }, }); - - const stats = Object.keys(metrics[0]).reduce((obj, key) => { - obj[key] = { - value: Number(metrics[0][key]) || 0, - change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0, - }; - return obj; - }, {}); + let stats: object = { + _id: '', + pageviews: 0, + uniques: 0, + bounces: 0, + totaltime: 0, + }; + if (metrics.length != 0) { + stats = Object.keys(metrics[0]).reduce((obj, key) => { + obj[key] = { + value: Number(metrics[0][key]) || 0, + change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0, + }; + return obj; + }, {}); + } return ok(res, stats); } diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts index 32c4c3d0..12efd6b2 100644 --- a/queries/analytics/event/getEventMetrics.ts +++ b/queries/analytics/event/getEventMetrics.ts @@ -45,14 +45,107 @@ async function relationalQuery( }; }, ) { - const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma; + const { getDatabaseType, toUuid, rawQuery, getDateQuery, getFilterQuery, client } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const filterQuery = getFilterQuery(filters, params); + const db = getDatabaseType(); - return rawQuery( - `select + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.customEvent], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $project: { + t: { + $dateTrunc: { + date: '$created_at', + unit: unit, + timezone: timezone, + }, + }, + event_name: 1, + }, + }, + { + $group: { + _id: { + t: '$t', + name: '$event_name', + }, + y: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id.name', + t: { + $dateToString: { + date: '$_id.t', + format: getDateQuery('', unit, timezone), + timezone: timezone, + }, + }, + y: 1, + _id: 0, + }, + }, + { + $sort: { + t: 1, + }, + }, + ], + }); + } else { + return rawQuery( + `select event_name x, ${getDateQuery('created_at', unit, timezone)} t, count(*) y @@ -64,8 +157,9 @@ async function relationalQuery( ${filterQuery} group by 1, 2 order by 2`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts index 73a71386..e06a567d 100644 --- a/queries/analytics/eventData/getEventData.ts +++ b/queries/analytics/eventData/getEventData.ts @@ -47,13 +47,123 @@ async function relationalQuery( }, ) { const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; - const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; + const { getDatabaseType, toUuid, rawQuery, getEventDataFilterQuery, getDateQuery, client } = + prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate, eventName || '']; - return rawQuery( - `select + if (db === 'mongodb') { + let joinAggregation: any = { match: {} }; + let matchAggregation: any = { match: {} }; + let eventTypeProjectProperty = ''; + let urlProjectProperty = ''; + if (eventName || urlPath) { + joinAggregation = { + $lookup: { + from: 'website_event', + localField: 'website_event_id', + foreignField: '_id', + as: 'result', + }, + }; + eventTypeProjectProperty = 'event_name: {$arrayElemAt: ["$result.event_name", 0]}'; + } + if (eventName) { + matchAggregation = { + $match: { + 'result.event_name': eventName, + }, + }; + } + if (urlPath) { + urlProjectProperty = 'url_path: {$arrayElemAt: ["$result.url_path", 0],}'; + } + let timeProjectProperty = ''; + if (timeSeries) { + timeProjectProperty = `t: $dateTrunc: {date: "$created_at",unit: ${timeSeries.unit}, timezone : ${timeSeries.timezone}`; + } + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + joinAggregation, + matchAggregation, + { + $project: { + eventTypeProjectProperty, + timeProjectProperty, + urlProjectProperty, + }, + }, + { + $group: { + _id: { + url_path: '$url_path', + event_name: '$event_name', + t: '$t', + }, + x: { + $sum: 1, + }, + }, + }, + { + $project: { + url_path: '$_id.url_path', + urlPath: '$_id.url_path', + event_name: '$_id.event_name', + eventName: '$_id.event_name', + x: 1, + t: '$_id.t', + _id: 0, + }, + }, + ], + }); + } else { + return rawQuery( + `select count(*) x ${eventName ? `,event_name eventName` : ''} ${urlPath ? `,url_path urlPath` : ''} @@ -72,8 +182,9 @@ async function relationalQuery( ${eventName ? `and eventName = $5` : ''} ${getEventDataFilterQuery(filters, params)} ${timeSeries ? 'group by t' : ''}`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts index db82720d..8f819396 100644 --- a/queries/analytics/pageview/getPageviewMetrics.ts +++ b/queries/analytics/pageview/getPageviewMetrics.ts @@ -31,7 +31,8 @@ async function relationalQuery( }, ) { const { startDate, endDate, filters = {}, column } = criteria; - const { rawQuery, parseFilters, toUuid } = prisma; + const { getDatabaseType, rawQuery, parseFilters, toUuid, client } = prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [ @@ -43,16 +44,99 @@ async function relationalQuery( ]; let excludeDomain = ''; + let excludeDomainMongo = {}; if (column === 'referrer_domain') { excludeDomain = 'and website_event.referrer_domain != $6'; + excludeDomainMongo = { + $ne: ['$referrer_domain', website.domain], + }; params.push(website.domain); } const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select ${column} x, count(*) y + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', params[4]], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + excludeDomainMongo, + ], + }, + }, + }, + { + $group: { + _id: '$' + column, + y: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id', + y: 1, + _id: 0, + }, + }, + { + $sort: { + x: 1, + }, + }, + { + $sort: { + y: -1, + }, + }, + { + $limit: 100, + }, + ], + }); + } else { + return rawQuery( + `select ${column} x, count(*) y from website_event ${joinSession} where website_event.website_id = $1${toUuid()} @@ -64,8 +148,9 @@ async function relationalQuery( group by 1 order by 2 desc limit 100`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index 9971fc37..2055391a 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -52,20 +52,112 @@ async function relationalQuery( const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - //TODO: 구현해야 함 - + let sessionInclude = ''; + let sessionGroupAggregation: any = { $match: {} }; + let sessionProjectAggregation: any = { $match: {} }; + if (count !== '*') { + sessionInclude = 'session_id : 1'; + sessionGroupAggregation = { + $group: { + _id: { + t: '$t', + session_id: '$session_id', + }, + }, + }; + sessionProjectAggregation = { + $project: { + t: '$_id.t', + }, + }; + } if (db === 'mongodb') { return await client.websiteEvent.aggregateRaw({ pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.pageView], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $project: { + t: { + $dateTrunc: { + date: '$created_at', + unit: unit, + }, + }, + sessionInclude, + }, + }, + sessionGroupAggregation, + sessionProjectAggregation, + { + $group: { + _id: { + t: '$t', + session_id: '$session_id', + }, + y: { + $sum: 1, + }, + }, + }, { $project: { x: { $dateToString: { - date: '$created_at', + date: '$_id.t', + format: getDateQuery('', unit, timezone), timezone: timezone, - format: getDateQuery('website_event.created_at', unit, timezone), }, }, + y: 1, + _id: 0, + }, + }, + { + $sort: { + x: 1, }, }, ], diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts index 57d4a736..bba0280a 100644 --- a/queries/analytics/session/getSessionMetrics.ts +++ b/queries/analytics/session/getSessionMetrics.ts @@ -23,12 +23,103 @@ async function relationalQuery( const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const { startDate, endDate, column, filters = {} } = criteria; - const { toUuid, parseFilters, rawQuery } = prisma; + const { getDatabaseType, toUuid, parseFilters, rawQuery, client } = prisma; + const db = getDatabaseType(); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select ${column} x, count(*) y + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $group: { + _id: '$session_id', + }, + }, + { + $lookup: { + from: 'session', + localField: '_id', + foreignField: '_id', + as: 'session', + }, + }, + { + $project: { + session: { + $arrayElemAt: ['$session', 0], + }, + }, + }, + { + $group: { + _id: '$session.' + column, + sum: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id', + y: '$sum', + _id: 0, + }, + }, + { + $sort: { + sum: -1, + }, + }, + { + $limit: 100, + }, + ], + }); + } else { + return rawQuery( + `select ${column} x, count(*) y from session as x where x.session_id in ( select website_event.session_id @@ -44,10 +135,10 @@ async function relationalQuery( group by 1 order by 2 desc limit 100`, - params, - ); + params, + ); + } } - async function clickhouseQuery( websiteId: string, data: { startDate: Date; endDate: Date; column: string; filters: object }, diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 4b8ce5ec..696837fc 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -39,6 +39,50 @@ async function relationalQuery( if (db === 'mongodb') { return await client.websiteEvent.aggregateRaw({ pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.pageView], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, { $project: { session_id: '$session_id', From 555e5afd15e1540365a53a4efe0d126379161b80 Mon Sep 17 00:00:00 2001 From: minkik Date: Thu, 18 May 2023 10:35:24 +0900 Subject: [PATCH 4/7] fix: operator bug --- queries/analytics/event/getEventMetrics.ts | 2 +- queries/analytics/eventData/getEventData.ts | 2 +- queries/analytics/eventData/saveEventData.ts | 4 +++- queries/analytics/pageview/getPageviewMetrics.ts | 2 +- queries/analytics/pageview/getPageviewStats.ts | 2 +- queries/analytics/session/getSessionMetrics.ts | 2 +- queries/analytics/stats/getWebsiteStats.ts | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts index 12efd6b2..c2bf2c08 100644 --- a/queries/analytics/event/getEventMetrics.ts +++ b/queries/analytics/event/getEventMetrics.ts @@ -86,7 +86,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts index e06a567d..5b1f7930 100644 --- a/queries/analytics/eventData/getEventData.ts +++ b/queries/analytics/eventData/getEventData.ts @@ -114,7 +114,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index 90e63565..af6f2ace 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -30,7 +30,9 @@ async function relationalQuery(data: { const { websiteId, eventId, eventData } = data; const jsonKeys = flattenJSON(eventData); - + if (jsonKeys.length == 0) { + return Promise.resolve({ count: 0 }); + } //id, websiteEventId, eventStringValue const flattendData = jsonKeys.map(a => ({ id: uuid(), diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts index 8f819396..b7133833 100644 --- a/queries/analytics/pageview/getPageviewMetrics.ts +++ b/queries/analytics/pageview/getPageviewMetrics.ts @@ -90,7 +90,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index 2055391a..a6847ea7 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -105,7 +105,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts index bba0280a..72d8ce70 100644 --- a/queries/analytics/session/getSessionMetrics.ts +++ b/queries/analytics/session/getSessionMetrics.ts @@ -59,7 +59,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 696837fc..dbe43b5c 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -70,7 +70,7 @@ async function relationalQuery( ], }, { - lte: [ + $lte: [ '$created_at', { $dateFromString: { From c5d3369c8c51040d689a6bc2c17f899a49a238e5 Mon Sep 17 00:00:00 2001 From: minkik Date: Thu, 18 May 2023 14:29:54 +0900 Subject: [PATCH 5/7] fix: bug fix --- pages/api/websites/[id]/stats.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pages/api/websites/[id]/stats.ts b/pages/api/websites/[id]/stats.ts index a61ba302..bf111d6b 100644 --- a/pages/api/websites/[id]/stats.ts +++ b/pages/api/websites/[id]/stats.ts @@ -75,7 +75,7 @@ export default async ( city, }, }); - const prevPeriod = await getWebsiteStats(websiteId, { + let prevPeriod = await getWebsiteStats(websiteId, { startDate: prevStartDate, endDate: prevEndDate, filters: { @@ -92,8 +92,17 @@ export default async ( city, }, }); + if (prevPeriod.length === 0) { + prevPeriod = [ + { + pageviews: 0, + uniques: 0, + bounces: 0, + totaltime: 0, + }, + ]; + } let stats: object = { - _id: '', pageviews: 0, uniques: 0, bounces: 0, From 4c57ab138818e190dc6acb661927e2263a36be9e Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Thu, 18 May 2023 18:24:42 +0900 Subject: [PATCH 6/7] fix: use buildInfo instead of serverStatus because of authorization problem --- scripts/check-db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-db.js b/scripts/check-db.js index a01c5180..b5d322d2 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -55,7 +55,7 @@ async function checkDatabaseVersion(databaseType) { let version; if (databaseType === 'mongodb') { const query = await prisma.$runCommandRaw({ - serverStatus: 1, + buildInfo: 1, }); version = semver.valid(query.version); } else { From b5b689b1565b88e41922e5a00750cc6c0866599f Mon Sep 17 00:00:00 2001 From: minkik Date: Fri, 19 May 2023 14:10:53 +0900 Subject: [PATCH 7/7] feat: separate mongoQuery & add mongo filter --- lib/db.js | 5 +- lib/prisma.ts | 20 ++ queries/analytics/event/getEventMetrics.ts | 225 ++++++++------- queries/analytics/event/getEvents.ts | 4 +- queries/analytics/event/saveEvent.ts | 3 +- queries/analytics/eventData/getEventData.ts | 258 +++++++++-------- queries/analytics/eventData/saveEventData.ts | 3 +- .../analytics/pageview/getPageviewMetrics.ts | 209 ++++++++------ .../analytics/pageview/getPageviewStats.ts | 272 ++++++++++-------- queries/analytics/session/createSession.ts | 3 +- queries/analytics/session/getSession.ts | 3 +- .../analytics/session/getSessionMetrics.ts | 205 ++++++------- queries/analytics/session/getSessions.ts | 3 +- queries/analytics/stats/getActiveVisitors.ts | 100 +++---- queries/analytics/stats/getWebsiteStats.ts | 255 ++++++++-------- 15 files changed, 858 insertions(+), 710 deletions(-) diff --git a/lib/db.js b/lib/db.js index d85b8aa7..07f98849 100644 --- a/lib/db.js +++ b/lib/db.js @@ -26,9 +26,12 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { export async function runQuery(queries) { const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL); - if (db === POSTGRESQL || db === MYSQL || db === MONGODB) { + if (db === POSTGRESQL || db === MYSQL) { return queries[PRISMA](); } + if (db === MONGODB) { + return queries[MONGODB](); + } if (db === CLICKHOUSE) { if (queries[KAFKA]) { diff --git a/lib/prisma.ts b/lib/prisma.ts index cf6ec9fe..ee5ace9e 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -142,6 +142,25 @@ function parseFilters( }; } +function parseMongoFilter(filters: { [key: string]: any } = {}) { + const query = {}; + + for (let k in filters) { + const v = filters[k]; + if (v !== undefined) { + const tempK = FILTER_COLUMNS[k]; + if (tempK !== undefined) { + k = tempK; + } + if (k === 'browser' || k === 'os' || k === 'device' || k === 'language') { + k = 'session.' + k; + } + query[k] = v; + } + } + return { $match: query }; +} + async function rawQuery(query: string, params: never[] = []): Promise { const db = getDatabaseType(process.env.DATABASE_URL); @@ -163,5 +182,6 @@ export default { getEventDataFilterQuery, toUuid, parseFilters, + parseMongoFilter, rawQuery, }; diff --git a/queries/analytics/event/getEventMetrics.ts b/queries/analytics/event/getEventMetrics.ts index c2bf2c08..1b81fd65 100644 --- a/queries/analytics/event/getEventMetrics.ts +++ b/queries/analytics/event/getEventMetrics.ts @@ -1,6 +1,6 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; import { WebsiteEventMetric } from 'lib/types'; import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; @@ -23,6 +23,7 @@ export async function getEventMetrics( return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), + [MONGODB]: () => mongodbQuery(...args), }); } @@ -45,107 +46,13 @@ async function relationalQuery( }; }, ) { - const { getDatabaseType, toUuid, rawQuery, getDateQuery, getFilterQuery, client } = prisma; + const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const filterQuery = getFilterQuery(filters, params); - const db = getDatabaseType(); - - if (db === 'mongodb') { - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$event_type', EVENT_TYPE.customEvent], - }, - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - { - $project: { - t: { - $dateTrunc: { - date: '$created_at', - unit: unit, - timezone: timezone, - }, - }, - event_name: 1, - }, - }, - { - $group: { - _id: { - t: '$t', - name: '$event_name', - }, - y: { - $sum: 1, - }, - }, - }, - { - $project: { - x: '$_id.name', - t: { - $dateToString: { - date: '$_id.t', - format: getDateQuery('', unit, timezone), - timezone: timezone, - }, - }, - y: 1, - _id: 0, - }, - }, - { - $sort: { - t: 1, - }, - }, - ], - }); - } else { - return rawQuery( - `select + return rawQuery( + `select event_name x, ${getDateQuery('created_at', unit, timezone)} t, count(*) y @@ -157,9 +64,8 @@ async function relationalQuery( ${filterQuery} group by 1, 2 order by 2`, - params, - ); - } + params, + ); } async function clickhouseQuery( @@ -202,3 +108,120 @@ async function clickhouseQuery( params, ); } + +async function mongodbQuery( + websiteId: string, + { + startDate, + endDate, + timezone = 'utc', + unit = 'day', + filters, + }: { + startDate: Date; + endDate: Date; + timezone: string; + unit: string; + filters: { + url: string; + eventName: string; + }; + }, +) { + const { getDateQuery, parseMongoFilter, client } = prisma; + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const mongoFilter = parseMongoFilter(filters); + + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.customEvent], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $project: { + t: { + $dateTrunc: { + date: '$created_at', + unit: unit, + timezone: timezone, + }, + }, + event_name: 1, + }, + }, + { + $group: { + _id: { + t: '$t', + name: '$event_name', + }, + y: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id.name', + t: { + $dateToString: { + date: '$_id.t', + format: getDateQuery('', unit, timezone), + timezone: timezone, + }, + }, + y: 1, + _id: 0, + }, + }, + { + $sort: { + t: 1, + }, + }, + ], + }); +} diff --git a/queries/analytics/event/getEvents.ts b/queries/analytics/event/getEvents.ts index b3853f2d..2a9e585b 100644 --- a/queries/analytics/event/getEvents.ts +++ b/queries/analytics/event/getEvents.ts @@ -1,11 +1,11 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { EVENT_TYPE } from 'lib/constants'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; export function getEvents(...args: [websiteId: string, startAt: Date, eventType: number]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index 9a7db00d..2b003797 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -1,5 +1,5 @@ import { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE } from 'lib/constants'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; import { uuid } from 'lib/crypto'; @@ -29,6 +29,7 @@ export async function saveEvent(args: { }) { return runQuery({ [PRISMA]: () => relationalQuery(args), + [MONGODB]: () => relationalQuery(args), [CLICKHOUSE]: () => clickhouseQuery(args), }); } diff --git a/queries/analytics/eventData/getEventData.ts b/queries/analytics/eventData/getEventData.ts index 5b1f7930..83a6468d 100644 --- a/queries/analytics/eventData/getEventData.ts +++ b/queries/analytics/eventData/getEventData.ts @@ -1,5 +1,5 @@ import clickhouse from 'lib/clickhouse'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { WebsiteEventDataMetric } from 'lib/types'; import { loadWebsite } from 'lib/query'; @@ -23,6 +23,7 @@ export async function getEventData( ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongoQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -47,123 +48,13 @@ async function relationalQuery( }, ) { const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; - const { getDatabaseType, toUuid, rawQuery, getEventDataFilterQuery, getDateQuery, client } = - prisma; - const db = getDatabaseType(); + const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate, eventName || '']; - if (db === 'mongodb') { - let joinAggregation: any = { match: {} }; - let matchAggregation: any = { match: {} }; - let eventTypeProjectProperty = ''; - let urlProjectProperty = ''; - if (eventName || urlPath) { - joinAggregation = { - $lookup: { - from: 'website_event', - localField: 'website_event_id', - foreignField: '_id', - as: 'result', - }, - }; - eventTypeProjectProperty = 'event_name: {$arrayElemAt: ["$result.event_name", 0]}'; - } - if (eventName) { - matchAggregation = { - $match: { - 'result.event_name': eventName, - }, - }; - } - if (urlPath) { - urlProjectProperty = 'url_path: {$arrayElemAt: ["$result.url_path", 0],}'; - } - let timeProjectProperty = ''; - if (timeSeries) { - timeProjectProperty = `t: $dateTrunc: {date: "$created_at",unit: ${timeSeries.unit}, timezone : ${timeSeries.timezone}`; - } - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - joinAggregation, - matchAggregation, - { - $project: { - eventTypeProjectProperty, - timeProjectProperty, - urlProjectProperty, - }, - }, - { - $group: { - _id: { - url_path: '$url_path', - event_name: '$event_name', - t: '$t', - }, - x: { - $sum: 1, - }, - }, - }, - { - $project: { - url_path: '$_id.url_path', - urlPath: '$_id.url_path', - event_name: '$_id.event_name', - eventName: '$_id.event_name', - x: 1, - t: '$_id.t', - _id: 0, - }, - }, - ], - }); - } else { - return rawQuery( - `select + return rawQuery( + `select count(*) x ${eventName ? `,event_name eventName` : ''} ${urlPath ? `,url_path urlPath` : ''} @@ -182,9 +73,8 @@ async function relationalQuery( ${eventName ? `and eventName = $5` : ''} ${getEventDataFilterQuery(filters, params)} ${timeSeries ? 'group by t' : ''}`, - params, - ); - } + params, + ); } async function clickhouseQuery( @@ -231,3 +121,137 @@ async function clickhouseQuery( params, ); } + +async function mongoQuery( + websiteId: string, + data: { + startDate: Date; + endDate: Date; + timeSeries?: { + unit: string; + timezone: string; + }; + eventName: string; + urlPath?: string; + filters: [ + { + eventKey?: string; + eventValue?: string | number | boolean | Date; + }, + ]; + }, +) { + const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; + const { client, parseMongoFilter } = prisma; + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const mongoFilter = parseMongoFilter(filters); + + let joinAggregation: any = { match: {} }; + let matchAggregation: any = { match: {} }; + let eventTypeProjectProperty = ''; + let urlProjectProperty = ''; + if (eventName || urlPath) { + joinAggregation = { + $lookup: { + from: 'website_event', + localField: 'website_event_id', + foreignField: '_id', + as: 'result', + }, + }; + eventTypeProjectProperty = 'event_name: {$arrayElemAt: ["$result.event_name", 0]}'; + } + if (eventName) { + matchAggregation = { + $match: { + 'result.event_name': eventName, + }, + }; + } + if (urlPath) { + urlProjectProperty = 'url_path: {$arrayElemAt: ["$result.url_path", 0],}'; + } + let timeProjectProperty = ''; + if (timeSeries) { + timeProjectProperty = `t: $dateTrunc: {date: "$created_at",unit: ${timeSeries.unit}, timezone : ${timeSeries.timezone}`; + } + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + joinAggregation, + matchAggregation, + { + $project: { + eventTypeProjectProperty, + timeProjectProperty, + urlProjectProperty, + }, + }, + { + $group: { + _id: { + url_path: '$url_path', + event_name: '$event_name', + t: '$t', + }, + x: { + $sum: 1, + }, + }, + }, + { + $project: { + url_path: '$_id.url_path', + urlPath: '$_id.url_path', + event_name: '$_id.event_name', + eventName: '$_id.event_name', + x: 1, + t: '$_id.t', + _id: 0, + }, + }, + ], + }); +} diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index af6f2ace..a291f6f3 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -1,7 +1,7 @@ import { Prisma } from '@prisma/client'; import { EVENT_DATA_TYPE } from 'lib/constants'; import { uuid } from 'lib/crypto'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import { flattenJSON } from 'lib/eventData'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; @@ -18,6 +18,7 @@ export async function saveEventData(args: { }) { return runQuery({ [PRISMA]: () => relationalQuery(args), + [MONGODB]: () => relationalQuery(args), [CLICKHOUSE]: () => clickhouseQuery(args), }); } diff --git a/queries/analytics/pageview/getPageviewMetrics.ts b/queries/analytics/pageview/getPageviewMetrics.ts index b7133833..0aadd7e6 100644 --- a/queries/analytics/pageview/getPageviewMetrics.ts +++ b/queries/analytics/pageview/getPageviewMetrics.ts @@ -1,6 +1,6 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; @@ -17,6 +17,7 @@ export async function getPageviewMetrics( ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongodbQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -31,8 +32,7 @@ async function relationalQuery( }, ) { const { startDate, endDate, filters = {}, column } = criteria; - const { getDatabaseType, rawQuery, parseFilters, toUuid, client } = prisma; - const db = getDatabaseType(); + const { rawQuery, parseFilters, toUuid } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [ @@ -44,99 +44,15 @@ async function relationalQuery( ]; let excludeDomain = ''; - let excludeDomainMongo = {}; if (column === 'referrer_domain') { excludeDomain = 'and website_event.referrer_domain != $6'; - excludeDomainMongo = { - $ne: ['$referrer_domain', website.domain], - }; params.push(website.domain); } const { filterQuery, joinSession } = parseFilters(filters, params); - - if (db === 'mongodb') { - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$event_type', params[4]], - }, - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - excludeDomainMongo, - ], - }, - }, - }, - { - $group: { - _id: '$' + column, - y: { - $sum: 1, - }, - }, - }, - { - $project: { - x: '$_id', - y: 1, - _id: 0, - }, - }, - { - $sort: { - x: 1, - }, - }, - { - $sort: { - y: -1, - }, - }, - { - $limit: 100, - }, - ], - }); - } else { - return rawQuery( - `select ${column} x, count(*) y + return rawQuery( + `select ${column} x, count(*) y from website_event ${joinSession} where website_event.website_id = $1${toUuid()} @@ -148,9 +64,8 @@ async function relationalQuery( group by 1 order by 2 desc limit 100`, - params, - ); - } + params, + ); } async function clickhouseQuery( @@ -196,3 +111,113 @@ async function clickhouseQuery( params, ); } + +async function mongodbQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + column: string; + filters: object; + }, +) { + const { startDate, endDate, filters = {}, column } = criteria; + const { parseMongoFilter, client } = prisma; + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const params: any = [ + websiteId, + resetDate, + startDate, + endDate, + column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + ]; + + let excludeDomainMongo: any = ''; + + if (column === 'referrer_domain') { + excludeDomainMongo = { + $ne: ['$referrer_domain', website.domain], + }; + params.push(website.domain); + } + const mongoFilter = parseMongoFilter(filters); + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', params[4]], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + excludeDomainMongo, + ], + }, + }, + }, + { + $group: { + _id: '$' + column, + y: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id', + y: 1, + _id: 0, + }, + }, + { + $sort: { + x: 1, + }, + }, + { + $sort: { + y: -1, + }, + }, + { + $limit: 100, + }, + ], + }); +} diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index a6847ea7..1df5455b 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -1,5 +1,5 @@ import clickhouse from 'lib/clickhouse'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; @@ -20,6 +20,7 @@ export async function getPageviewStats( ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongodbQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -45,126 +46,14 @@ async function relationalQuery( filters = {}, sessionKey = 'session_id', } = criteria; - const { getDatabaseType, toUuid, getDateQuery, parseFilters, rawQuery, client } = prisma; - const db = getDatabaseType(); + const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - let sessionInclude = ''; - let sessionGroupAggregation: any = { $match: {} }; - let sessionProjectAggregation: any = { $match: {} }; - if (count !== '*') { - sessionInclude = 'session_id : 1'; - sessionGroupAggregation = { - $group: { - _id: { - t: '$t', - session_id: '$session_id', - }, - }, - }; - sessionProjectAggregation = { - $project: { - t: '$_id.t', - }, - }; - } - if (db === 'mongodb') { - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$event_type', EVENT_TYPE.pageView], - }, - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - { - $project: { - t: { - $dateTrunc: { - date: '$created_at', - unit: unit, - }, - }, - sessionInclude, - }, - }, - sessionGroupAggregation, - sessionProjectAggregation, - { - $group: { - _id: { - t: '$t', - session_id: '$session_id', - }, - y: { - $sum: 1, - }, - }, - }, - { - $project: { - x: { - $dateToString: { - date: '$_id.t', - format: getDateQuery('', unit, timezone), - timezone: timezone, - }, - }, - y: 1, - _id: 0, - }, - }, - { - $sort: { - x: 1, - }, - }, - ], - }); - } else { - return rawQuery( - `select ${getDateQuery('website_event.created_at', unit, timezone)} x, + return rawQuery( + `select ${getDateQuery('website_event.created_at', unit, timezone)} x, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from website_event ${joinSession} @@ -174,9 +63,8 @@ async function relationalQuery( and event_type = ${EVENT_TYPE.pageView} ${filterQuery} group by 1`, - params, - ); - } + params, + ); } async function clickhouseQuery( @@ -231,3 +119,149 @@ async function clickhouseQuery( params, ); } + +async function mongodbQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + timezone?: string; + unit?: string; + count?: string; + filters: object; + sessionKey?: string; + }, +) { + const { + startDate, + endDate, + timezone = 'utc', + unit = 'day', + count = '*', + filters = {}, + } = criteria; + const { getDateQuery, client, parseMongoFilter } = prisma; + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const mongoFilter = parseMongoFilter(filters); + let sessionInclude = ''; + let sessionGroupAggregation: any = { $match: {} }; + const sessionLookUpAggregation: any = { + $lookup: { + from: 'session', + foreignField: '_id', + localField: 'session_id', + as: 'session', + }, + }; + let sessionProjectAggregation: any = { $match: {} }; + + if (count !== '*') { + sessionInclude = 'session_id : 1'; + sessionGroupAggregation = { + $group: { + _id: { + t: '$t', + session_id: '$session_id', + }, + }, + }; + sessionProjectAggregation = { + $project: { + t: '$_id.t', + }, + }; + } + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + sessionLookUpAggregation, + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.pageView], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $project: { + t: { + $dateTrunc: { + date: '$created_at', + unit: unit, + }, + }, + sessionInclude, + }, + }, + sessionGroupAggregation, + sessionProjectAggregation, + { + $group: { + _id: { + t: '$t', + session_id: '$session_id', + }, + y: { + $sum: 1, + }, + }, + }, + { + $project: { + x: { + $dateToString: { + date: '$_id.t', + format: getDateQuery('', unit, timezone), + timezone: timezone, + }, + }, + y: 1, + _id: 0, + }, + }, + { + $sort: { + x: 1, + }, + }, + ], + }); +} diff --git a/queries/analytics/session/createSession.ts b/queries/analytics/session/createSession.ts index 22f7892f..7ba157ae 100644 --- a/queries/analytics/session/createSession.ts +++ b/queries/analytics/session/createSession.ts @@ -1,4 +1,4 @@ -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; import cache from 'lib/cache'; @@ -7,6 +7,7 @@ import { Prisma } from '@prisma/client'; export async function createSession(args: Prisma.SessionCreateInput) { return runQuery({ [PRISMA]: () => relationalQuery(args), + [MONGODB]: () => relationalQuery(args), [CLICKHOUSE]: () => clickhouseQuery(args), }).then(async data => { if (cache.enabled) { diff --git a/queries/analytics/session/getSession.ts b/queries/analytics/session/getSession.ts index d226e832..f51b741c 100644 --- a/queries/analytics/session/getSession.ts +++ b/queries/analytics/session/getSession.ts @@ -1,11 +1,12 @@ import clickhouse from 'lib/clickhouse'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import { CLICKHOUSE, MONGODB, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { Prisma } from '@prisma/client'; export async function getSession(args: { id: string }) { return runQuery({ [PRISMA]: () => relationalQuery(args), + [MONGODB]: () => relationalQuery(args), [CLICKHOUSE]: () => clickhouseQuery(args), }); } diff --git a/queries/analytics/session/getSessionMetrics.ts b/queries/analytics/session/getSessionMetrics.ts index 72d8ce70..315f9b89 100644 --- a/queries/analytics/session/getSessionMetrics.ts +++ b/queries/analytics/session/getSessionMetrics.ts @@ -1,6 +1,6 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; @@ -12,6 +12,7 @@ export async function getSessionMetrics( ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongodbQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -23,103 +24,12 @@ async function relationalQuery( const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const { startDate, endDate, column, filters = {} } = criteria; - const { getDatabaseType, toUuid, parseFilters, rawQuery, client } = prisma; - const db = getDatabaseType(); + const { toUuid, parseFilters, rawQuery } = prisma; const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - if (db === 'mongodb') { - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - { - $group: { - _id: '$session_id', - }, - }, - { - $lookup: { - from: 'session', - localField: '_id', - foreignField: '_id', - as: 'session', - }, - }, - { - $project: { - session: { - $arrayElemAt: ['$session', 0], - }, - }, - }, - { - $group: { - _id: '$session.' + column, - sum: { - $sum: 1, - }, - }, - }, - { - $project: { - x: '$_id', - y: '$sum', - _id: 0, - }, - }, - { - $sort: { - sum: -1, - }, - }, - { - $limit: 100, - }, - ], - }); - } else { - return rawQuery( - `select ${column} x, count(*) y + return rawQuery( + `select ${column} x, count(*) y from session as x where x.session_id in ( select website_event.session_id @@ -135,9 +45,8 @@ async function relationalQuery( group by 1 order by 2 desc limit 100`, - params, - ); - } + params, + ); } async function clickhouseQuery( websiteId: string, @@ -164,3 +73,103 @@ async function clickhouseQuery( params, ); } + +async function mongodbQuery( + websiteId: string, + criteria: { startDate: Date; endDate: Date; column: string; filters: object }, +) { + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const { startDate, endDate, column, filters = {} } = criteria; + const { parseMongoFilter, client } = prisma; + const mongoFilter = parseMongoFilter(filters); + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $group: { + _id: '$session_id', + }, + }, + { + $lookup: { + from: 'session', + localField: '_id', + foreignField: '_id', + as: 'session', + }, + }, + { + $project: { + session: { + $arrayElemAt: ['$session', 0], + }, + }, + }, + { + $group: { + _id: '$session.' + column, + sum: { + $sum: 1, + }, + }, + }, + { + $project: { + x: '$_id', + y: '$sum', + _id: 0, + }, + }, + { + $sort: { + sum: -1, + }, + }, + { + $limit: 100, + }, + ], + }); +} diff --git a/queries/analytics/session/getSessions.ts b/queries/analytics/session/getSessions.ts index a4fbb501..10c6d496 100644 --- a/queries/analytics/session/getSessions.ts +++ b/queries/analytics/session/getSessions.ts @@ -1,10 +1,11 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db'; +import { runQuery, PRISMA, CLICKHOUSE, MONGODB } from 'lib/db'; export async function getSessions(...args: [websiteId: string, startAt: Date]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } diff --git a/queries/analytics/stats/getActiveVisitors.ts b/queries/analytics/stats/getActiveVisitors.ts index 5d38d81f..04efdfcd 100644 --- a/queries/analytics/stats/getActiveVisitors.ts +++ b/queries/analytics/stats/getActiveVisitors.ts @@ -1,72 +1,31 @@ import { subMinutes } from 'date-fns'; import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; export async function getActiveVisitors(...args: [websiteId: string]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongodbQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery(websiteId: string) { - const { getDatabaseType, toUuid, rawQuery, client } = prisma; - const db = getDatabaseType(); + const { toUuid, rawQuery } = prisma; const date = subMinutes(new Date(), 5); const params: any = [websiteId, date]; - if (db === 'mongodb') { - const result: any = await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: date.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - { - $group: { - _id: '$session_id', - }, - }, - { - $count: 'x', - }, - ], - }); - if (result.length > 0) { - return { x: result[0].x }; - } else { - return { x: 0 }; - } - } else { - return rawQuery( - `select count(distinct session_id) x + return rawQuery( + `select count(distinct session_id) x from website_event join website on website_event.website_id = website.website_id where website.website_id = $1${toUuid()} and website_event.created_at >= $2`, - params, - ); - } + params, + ); } async function clickhouseQuery(websiteId: string) { @@ -81,3 +40,48 @@ async function clickhouseQuery(websiteId: string) { params, ); } + +async function mongodbQuery(websiteId: string) { + const { client } = prisma; + + const date = subMinutes(new Date(), 5); + + const result: any = await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: date.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $group: { + _id: '$session_id', + }, + }, + { + $count: 'x', + }, + ], + }); + if (result.length > 0) { + return { x: result[0].x }; + } else { + return { x: 0 }; + } +} diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index dbe43b5c..d1d2ef8b 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -1,6 +1,6 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import { runQuery, CLICKHOUSE, PRISMA, MONGODB } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/query'; @@ -12,6 +12,7 @@ export async function getWebsiteStats( ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), + [MONGODB]: () => mongodbQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } @@ -21,133 +22,13 @@ async function relationalQuery( criteria: { startDate: Date; endDate: Date; filters: object }, ) { const { startDate, endDate, filters = {} } = criteria; - const { - getDatabaseType, - toUuid, - getDateQuery, - getTimestampInterval, - parseFilters, - rawQuery, - client, - } = prisma; - const db = getDatabaseType(); + const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - - if (db === 'mongodb') { - return await client.websiteEvent.aggregateRaw({ - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ['$event_type', EVENT_TYPE.pageView], - }, - { - $eq: ['$website_id', websiteId], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: resetDate.toISOString(), - }, - }, - ], - }, - { - $gte: [ - '$created_at', - { - $dateFromString: { - dateString: startDate.toISOString(), - }, - }, - ], - }, - { - $lte: [ - '$created_at', - { - $dateFromString: { - dateString: endDate.toISOString(), - }, - }, - ], - }, - ], - }, - }, - }, - { - $project: { - session_id: '$session_id', - hour: { - $toString: { $hour: '$created_at' }, - }, - created_at: '$created_at', - }, - }, - { - $group: { - _id: { - $concat: ['$session_id', ':', '$hour'], - }, - session_id: { $first: '$session_id' }, - hour: { $first: '$hour' }, - count: { $sum: 1 }, - timeMax: { $max: '$created_at' }, - timeMin: { $min: '$created_at' }, - }, - }, - { - $project: { - _id: '$_id', - session_id: '$session_id', - hour: '$hour', - count: '$count', - time: { - $dateDiff: { - endDate: '$timeMax', - startDate: '$timeMin', - unit: 'second', - }, - }, - bounce: { - $cond: { - if: { $eq: ['$count', 1] }, - then: 1, - else: 0, - }, - }, - }, - }, - { - $group: { - _id: '$session_id', - pageviews: { $sum: '$count' }, - bounces: { $sum: '$bounce' }, - totaltime: { $sum: '$time' }, - }, - }, - { - $group: { - _id: '', - pageviews: { $sum: '$pageviews' }, - uniques: { $sum: 1 }, - bounces: { $sum: '$bounces' }, - totaltime: { $sum: '$totaltime' }, - }, - }, - ], - }); - } else { - return rawQuery( - `select sum(t.c) as "pageviews", + return rawQuery( + `select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(t.time) as "totaltime" @@ -167,9 +48,8 @@ async function relationalQuery( ${filterQuery} group by 1, 2 ) t`, - params, - ); - } + params, + ); } async function clickhouseQuery( @@ -206,3 +86,124 @@ async function clickhouseQuery( params, ); } + +async function mongodbQuery( + websiteId: string, + criteria: { startDate: Date; endDate: Date; filters: object }, +) { + const { startDate, endDate, filters = {} } = criteria; + const { parseMongoFilter, client } = prisma; + const website = await loadWebsite(websiteId); + const resetDate = new Date(website?.resetAt || website?.createdAt); + const mongoFilter = parseMongoFilter(filters); + + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + mongoFilter, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$event_type', EVENT_TYPE.pageView], + }, + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: resetDate.toISOString(), + }, + }, + ], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: startDate.toISOString(), + }, + }, + ], + }, + { + $lte: [ + '$created_at', + { + $dateFromString: { + dateString: endDate.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $project: { + session_id: '$session_id', + hour: { + $toString: { $hour: '$created_at' }, + }, + created_at: '$created_at', + }, + }, + { + $group: { + _id: { + $concat: ['$session_id', ':', '$hour'], + }, + session_id: { $first: '$session_id' }, + hour: { $first: '$hour' }, + count: { $sum: 1 }, + timeMax: { $max: '$created_at' }, + timeMin: { $min: '$created_at' }, + }, + }, + { + $project: { + _id: '$_id', + session_id: '$session_id', + hour: '$hour', + count: '$count', + time: { + $dateDiff: { + endDate: '$timeMax', + startDate: '$timeMin', + unit: 'second', + }, + }, + bounce: { + $cond: { + if: { $eq: ['$count', 1] }, + then: 1, + else: 0, + }, + }, + }, + }, + { + $group: { + _id: '$session_id', + pageviews: { $sum: '$count' }, + bounces: { $sum: '$bounce' }, + totaltime: { $sum: '$time' }, + }, + }, + { + $group: { + _id: '', + pageviews: { $sum: '$pageviews' }, + uniques: { $sum: 1 }, + bounces: { $sum: '$bounces' }, + totaltime: { $sum: '$totaltime' }, + }, + }, + ], + }); +}