From bb30b43a8eb0201761a907df10078feac592b210 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 15 May 2023 15:49:21 +0900 Subject: [PATCH] feat: support mongodb connection --- db/mongodb/schema.prisma | 155 ++++++++++++++++++ lib/db.js | 5 +- lib/prisma.ts | 7 +- queries/admin/user.ts | 23 ++- .../analytics/pageview/getPageviewStats.ts | 31 +++- queries/analytics/stats/getActiveVisitors.ts | 62 +++++-- queries/analytics/stats/getWebsiteStats.ts | 86 +++++++++- scripts/check-db.js | 27 ++- scripts/copy-db-files.js | 4 +- 9 files changed, 370 insertions(+), 30 deletions(-) create mode 100644 db/mongodb/schema.prisma diff --git a/db/mongodb/schema.prisma b/db/mongodb/schema.prisma new file mode 100644 index 00000000..b6e463bc --- /dev/null +++ b/db/mongodb/schema.prisma @@ -0,0 +1,155 @@ +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "mongodb" + url = env("DATABASE_URL") + relationMode = "prisma" +} + +model User { + id String @id @map("_id") @db.String + username String @unique @db.String + password String @db.String + role String @map("role") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + deletedAt DateTime? @map("deleted_at") @db.Date + + website Website[] + teamUser TeamUser[] + + @@map("user") +} + +model Session { + id String @id @map("_id") @db.String + websiteId String @map("website_id") @db.String + hostname String? @db.String + browser String? @db.String + os String? @db.String + device String? @db.String + screen String? @db.String + language String? @db.String + country String? @db.String + subdivision1 String? @db.String + subdivision2 String? @db.String + city String? @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + websiteEvent WebsiteEvent[] + + @@index([createdAt]) + @@index([websiteId]) + @@map("session") +} + +model Website { + id String @id @map("_id") @db.String + name String @db.String + domain String? @db.String + shareId String? @unique @map("share_id") @db.String + resetAt DateTime? @map("reset_at") @db.Date + userId String? @map("user_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + deletedAt DateTime? @map("deleted_at") @db.Date + + user User? @relation(fields: [userId], references: [id]) + teamWebsite TeamWebsite[] + eventData EventData[] + + @@index([userId]) + @@index([createdAt]) + @@map("website") +} + +model WebsiteEvent { + id String @id() @map("_id") @db.String + websiteId String @map("website_id") @db.String + sessionId String @map("session_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + urlPath String @map("url_path") @db.String + urlQuery String? @map("url_query") @db.String + referrerPath String? @map("referrer_path") @db.String + referrerQuery String? @map("referrer_query") @db.String + referrerDomain String? @map("referrer_domain") @db.String + pageTitle String? @map("page_title") @db.String + eventType Int @default(1) @map("event_type") @db.Int + eventName String? @map("event_name") @db.String + + eventData EventData[] + session Session @relation(fields: [sessionId], references: [id]) + + @@index([createdAt]) + @@index([sessionId]) + @@index([websiteId]) + @@index([websiteId, createdAt]) + @@index([websiteId, sessionId, createdAt]) + @@map("website_event") +} + +model EventData { + id String @id() @map("_id") @db.String + websiteEventId String @map("website_event_id") @db.String + websiteId String @map("website_id") @db.String + eventKey String @map("event_key") @db.String + eventStringValue String? @map("event_string_value") @db.String + eventNumericValue Float? @map("event_numeric_value") @db.Double // (19, 4) + eventDateValue DateTime? @map("event_date_value") @db.Date + eventDataType Int @map("event_data_type") @db.Int + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + website Website @relation(fields: [websiteId], references: [id]) + websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) + + @@index([createdAt]) + @@index([websiteId]) + @@index([websiteEventId]) + @@index([websiteId, websiteEventId, createdAt]) + @@map("event_data") +} + +model Team { + id String @id() @map("_id") @db.String + name String @db.String + accessCode String? @unique @map("access_code") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + + teamUser TeamUser[] + teamWebsite TeamWebsite[] + + @@map("team") +} + +model TeamUser { + id String @id() @map("_id") @db.String + teamId String @map("team_id") @db.String + userId String @map("user_id") @db.String + role String @map("role") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + updatedAt DateTime? @updatedAt @map("updated_at") @db.Date + + team Team @relation(fields: [teamId], references: [id]) + user User @relation(fields: [userId], references: [id]) + + @@index([teamId]) + @@index([userId]) + @@map("team_user") +} + +model TeamWebsite { + id String @id() @map("_id") @db.String + teamId String @map("team_id") @db.String + websiteId String @map("website_id") @db.String + createdAt DateTime? @default(now()) @map("created_at") @db.Date + + team Team @relation(fields: [teamId], references: [id]) + website Website @relation(fields: [websiteId], references: [id]) + + @@index([teamId]) + @@index([websiteId]) + @@map("team_website") +} diff --git a/lib/db.js b/lib/db.js index 19e46a3d..d85b8aa7 100644 --- a/lib/db.js +++ b/lib/db.js @@ -4,6 +4,7 @@ export const MYSQL = 'mysql'; export const CLICKHOUSE = 'clickhouse'; export const KAFKA = 'kafka'; export const KAFKA_PRODUCER = 'kafka-producer'; +export const MONGODB = 'mongodb'; // Fixes issue with converting bigint values BigInt.prototype.toJSON = function () { @@ -15,6 +16,8 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return POSTGRESQL; + } else if (type === 'mongodb+srv') { + return MONGODB; } return type; @@ -23,7 +26,7 @@ export function getDatabaseType(url = process.env.DATABASE_URL) { export async function runQuery(queries) { const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL); - if (db === POSTGRESQL || db === MYSQL) { + if (db === POSTGRESQL || db === MYSQL || db === MONGODB) { return queries[PRISMA](); } diff --git a/lib/prisma.ts b/lib/prisma.ts index 0a10d981..cf6ec9fe 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -1,6 +1,6 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; -import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; +import { MYSQL, POSTGRESQL, getDatabaseType, MONGODB } from 'lib/db'; import { getEventDataType } from './eventData'; import { FILTER_COLUMNS } from './constants'; @@ -51,6 +51,10 @@ function getDateQuery(field: string, unit: string, timezone?: string): string { return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`; } + + if (db === MONGODB) { + return MYSQL_DATE_FORMATS[unit]; + } } function getTimestampInterval(field: string): string { @@ -152,6 +156,7 @@ async function rawQuery(query: string, params: never[] = []): Promise { export default { ...prisma, + getDatabaseType: () => getDatabaseType(process.env.DATABASE_URL), getDateQuery, getTimestampInterval, getFilterQuery, diff --git a/queries/admin/user.ts b/queries/admin/user.ts index 412c7785..2feb913d 100644 --- a/queries/admin/user.ts +++ b/queries/admin/user.ts @@ -3,6 +3,16 @@ import cache from 'lib/cache'; import { ROLES } from 'lib/constants'; import prisma from 'lib/prisma'; import { Website, User, Roles } from 'lib/types'; +import { getDatabaseType } from '../../lib/db'; + +function whereDeletedAtNotNull() { + const db = getDatabaseType(process.env.DATABASE_URL); + if (db === 'mongodb') { + return { isSet: false }; + } else { + return null; + } +} export async function getUser( where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput, @@ -11,7 +21,14 @@ export async function getUser( const { includePassword = false, showDeleted = false } = options; return prisma.client.user.findFirst({ - where: { ...where, ...(showDeleted ? {} : { deletedAt: null }) }, + where: { + ...where, + ...(showDeleted + ? {} + : { + deletedAt: whereDeletedAtNotNull(), + }), + }, select: { id: true, username: true, @@ -26,7 +43,7 @@ export async function getUsers(): Promise { return prisma.client.user.findMany({ take: 100, where: { - deletedAt: null, + deletedAt: whereDeletedAtNotNull(), }, orderBy: [ { @@ -76,7 +93,7 @@ export async function getUserWebsites(userId: string): Promise { return prisma.client.website.findMany({ where: { userId, - deletedAt: null, + deletedAt: whereDeletedAtNotNull(), }, orderBy: [ { diff --git a/queries/analytics/pageview/getPageviewStats.ts b/queries/analytics/pageview/getPageviewStats.ts index 01e4ab14..9971fc37 100644 --- a/queries/analytics/pageview/getPageviewStats.ts +++ b/queries/analytics/pageview/getPageviewStats.ts @@ -45,14 +45,34 @@ async function relationalQuery( filters = {}, sessionKey = 'session_id', } = criteria; - const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma; + const { getDatabaseType, toUuid, getDateQuery, parseFilters, rawQuery, client } = prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select ${getDateQuery('website_event.created_at', unit, timezone)} x, + //TODO: 구현해야 함 + + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $project: { + x: { + $dateToString: { + date: '$created_at', + timezone: timezone, + format: getDateQuery('website_event.created_at', unit, timezone), + }, + }, + }, + }, + ], + }); + } else { + return rawQuery( + `select ${getDateQuery('website_event.created_at', unit, timezone)} x, count(${count !== '*' ? `${count}${sessionKey}` : count}) y from website_event ${joinSession} @@ -62,8 +82,9 @@ async function relationalQuery( and event_type = ${EVENT_TYPE.pageView} ${filterQuery} group by 1`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/queries/analytics/stats/getActiveVisitors.ts b/queries/analytics/stats/getActiveVisitors.ts index 89f092c1..5d38d81f 100644 --- a/queries/analytics/stats/getActiveVisitors.ts +++ b/queries/analytics/stats/getActiveVisitors.ts @@ -11,20 +11,62 @@ export async function getActiveVisitors(...args: [websiteId: string]) { } async function relationalQuery(websiteId: string) { - const { toUuid, rawQuery } = prisma; + const { getDatabaseType, toUuid, rawQuery, client } = prisma; + const db = getDatabaseType(); const date = subMinutes(new Date(), 5); const params: any = [websiteId, date]; - return rawQuery( - `select count(distinct session_id) x - from website_event - join website - on website_event.website_id = website.website_id - where website.website_id = $1${toUuid()} - and website_event.created_at >= $2`, - params, - ); + if (db === 'mongodb') { + const result: any = await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $match: { + $expr: { + $and: [ + { + $eq: ['$website_id', websiteId], + }, + { + $gte: [ + '$created_at', + { + $dateFromString: { + dateString: date.toISOString(), + }, + }, + ], + }, + ], + }, + }, + }, + { + $group: { + _id: '$session_id', + }, + }, + { + $count: 'x', + }, + ], + }); + if (result.length > 0) { + return { x: result[0].x }; + } else { + return { x: 0 }; + } + } else { + return rawQuery( + `select count(distinct session_id) x + from website_event + join website + on website_event.website_id = website.website_id + where website.website_id = $1${toUuid()} + and website_event.created_at >= $2`, + params, + ); + } } async function clickhouseQuery(websiteId: string) { diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 0021e793..4b8ce5ec 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -21,14 +21,89 @@ async function relationalQuery( criteria: { startDate: Date; endDate: Date; filters: object }, ) { const { startDate, endDate, filters = {} } = criteria; - const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; + const { + getDatabaseType, + toUuid, + getDateQuery, + getTimestampInterval, + parseFilters, + rawQuery, + client, + } = prisma; + const db = getDatabaseType(); const website = await loadWebsite(websiteId); const resetDate = new Date(website?.resetAt || website?.createdAt); const params: any = [websiteId, resetDate, startDate, endDate]; const { filterQuery, joinSession } = parseFilters(filters, params); - return rawQuery( - `select sum(t.c) as "pageviews", + if (db === 'mongodb') { + return await client.websiteEvent.aggregateRaw({ + pipeline: [ + { + $project: { + session_id: '$session_id', + hour: { + $toString: { $hour: '$created_at' }, + }, + created_at: '$created_at', + }, + }, + { + $group: { + _id: { + $concat: ['$session_id', ':', '$hour'], + }, + session_id: { $first: '$session_id' }, + hour: { $first: '$hour' }, + count: { $sum: 1 }, + timeMax: { $max: '$created_at' }, + timeMin: { $min: '$created_at' }, + }, + }, + { + $project: { + _id: '$_id', + session_id: '$session_id', + hour: '$hour', + count: '$count', + time: { + $dateDiff: { + endDate: '$timeMax', + startDate: '$timeMin', + unit: 'second', + }, + }, + bounce: { + $cond: { + if: { $eq: ['$count', 1] }, + then: 1, + else: 0, + }, + }, + }, + }, + { + $group: { + _id: '$session_id', + pageviews: { $sum: '$count' }, + bounces: { $sum: '$bounce' }, + totaltime: { $sum: '$time' }, + }, + }, + { + $group: { + _id: '', + pageviews: { $sum: '$pageviews' }, + uniques: { $sum: 1 }, + bounces: { $sum: '$bounces' }, + totaltime: { $sum: '$totaltime' }, + }, + }, + ], + }); + } else { + return rawQuery( + `select sum(t.c) as "pageviews", count(distinct t.session_id) as "uniques", sum(case when t.c = 1 then 1 else 0 end) as "bounces", sum(t.time) as "totaltime" @@ -48,8 +123,9 @@ async function relationalQuery( ${filterQuery} group by 1, 2 ) t`, - params, - ); + params, + ); + } } async function clickhouseQuery( diff --git a/scripts/check-db.js b/scripts/check-db.js index 3fd3a908..acf912bf 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -15,6 +15,8 @@ function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return 'postgresql'; + } else if (type === 'mongodb+srv') { + return 'mongodb'; } return type; @@ -50,10 +52,19 @@ async function checkConnection() { } async function checkDatabaseVersion(databaseType) { - const query = await prisma.$queryRaw`select version() as version`; - const version = semver.valid(semver.coerce(query[0].version)); + let version; + if (databaseType === 'mongodb') { + const query = await prisma.$runCommandRaw({ + serverStatus: 1, + }); + version = semver.valid(query.version); + } else { + const query = await prisma.$queryRaw`select version() as version`; + version = semver.valid(semver.coerce(query[0].version)); + } - const minVersion = databaseType === 'postgresql' ? '9.4.0' : '5.7.0'; + const minVersion = + databaseType === 'postgresql' ? '9.4.0' : databaseType === 'mongodb' ? '3.0.0' : '5.7.0'; if (semver.lt(version, minVersion)) { throw new Error( @@ -65,6 +76,10 @@ async function checkDatabaseVersion(databaseType) { } async function checkV1Tables() { + if (databaseType === 'mongodb') { + // Ignore + } + try { await prisma.$queryRaw`select * from account limit 1`; @@ -78,7 +93,11 @@ async function checkV1Tables() { } async function applyMigration() { - console.log(execSync('prisma migrate deploy').toString()); + if (databaseType === 'mongodb') { + console.log(execSync('prisma db push').toString()); + } else { + console.log(execSync('prisma migrate deploy').toString()); + } success('Database is up to date.'); } diff --git a/scripts/copy-db-files.js b/scripts/copy-db-files.js index 15c34674..9bdd8506 100644 --- a/scripts/copy-db-files.js +++ b/scripts/copy-db-files.js @@ -9,6 +9,8 @@ function getDatabaseType(url = process.env.DATABASE_URL) { if (type === 'postgres') { return 'postgresql'; + } else if (type === 'mongodb+srv') { + return 'mongodb'; } return type; @@ -16,7 +18,7 @@ function getDatabaseType(url = process.env.DATABASE_URL) { const databaseType = getDatabaseType(); -if (!databaseType || !['mysql', 'postgresql'].includes(databaseType)) { +if (!databaseType || !['mysql', 'postgresql', 'mongodb'].includes(databaseType)) { throw new Error('Missing or invalid database'); }