From 73b0981788f6d78a097a20aad2bca257a5967420 Mon Sep 17 00:00:00 2001 From: 1900 Date: Tue, 6 Sep 2022 13:41:52 +0800 Subject: [PATCH 01/61] Update zh-CN.json --- lang/zh-CN.json | 1 + 1 file changed, 1 insertion(+) diff --git a/lang/zh-CN.json b/lang/zh-CN.json index b09718d4..f154e07c 100644 --- a/lang/zh-CN.json +++ b/lang/zh-CN.json @@ -62,6 +62,7 @@ "label.username": "用户名", "label.view-details": "查看更多", "label.websites": "网站", + "label.yesterday": "昨天", "message.active-users": "当前在线 {x} 人", "message.confirm-delete": "你确定要删除 {target} 吗？", "message.confirm-reset": "您确定要重置 {target} 的数据吗?", From 99914d6e98de4dea11e52fd389ef131fb14dba34 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 5 Sep 2022 22:47:45 -0700 Subject: [PATCH 02/61] Add retry connection to redis. Clean up up redis/session logic --- lib/kafka.js | 4 ++++ lib/redis.js | 7 ++++++- lib/session.js | 10 ++++------ queries/analytics/session/createSession.js | 2 +- queries/analytics/session/getSessionByUuid.js | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/kafka.js b/lib/kafka.js index 6d007009..6b6f315f 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -33,6 +33,8 @@ function getClient() { global[KAFKA] = client; } + log('Kafka initialized'); + return client; } @@ -44,6 +46,8 @@ async function getProducer() { global[KAFKA_PRODUCER] = producer; } + log('Kafka producer initialized'); + return producer; } diff --git a/lib/redis.js b/lib/redis.js index 92c8bdb9..63b513e6 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -13,7 +13,12 @@ function getClient() { return null; } - const redis = new Redis(process.env.REDIS_URL); + const redis = new Redis(process.env.REDIS_URL, { + retryStrategy(times) { + log(`Redis reconnecting attempt: ${times}`); + return 5000; + }, + }); if (process.env.NODE_ENV !== 'production') { global[REDIS] = redis; diff --git a/lib/session.js b/lib/session.js index 1b79f316..6e78333b 100644 --- a/lib/session.js +++ b/lib/session.js @@ -32,7 +32,7 @@ export async function getSession(req) { // Check if website exists if (redis.client) { - websiteId = await redis.client.get(`website:${website_uuid}`); + websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } // Check database if redis does not have @@ -49,23 +49,21 @@ export async function getSession(req) { const session_uuid = uuid(websiteId, hostname, ip, userAgent); - let sessionCreated = false; let sessionId = null; let session = null; // Check if session exists if (redis.client) { - sessionCreated = !!(await redis.client.get(`session:${session_uuid}`)); + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); } // Check database if redis does not have - if (!sessionCreated) { + if (!sessionId) { session = await getSessionByUuid(session_uuid); - sessionCreated = !!session; sessionId = session ? session.session_id : null; } - if (!sessionCreated) { + if (!sessionId) { try { session = await createSession(websiteId, { session_uuid, diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 8153b8f4..4e23dac1 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -23,7 +23,7 @@ async function relationalQuery(website_id, data) { }) .then(async res => { if (redis.client && res) { - await redis.client.set(`session:${res.session_uuid}`, 1); + await redis.client.set(`session:${res.session_uuid}`, res.session_id); } return res; diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 6412ca8c..911848dc 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -19,7 +19,7 @@ async function relationalQuery(session_uuid) { }) .then(async res => { if (redis.client && res) { - await redis.client.set(`session:${res.session_uuid}`, 1); + await redis.client.set(`session:${res.session_uuid}`, res.session_id); } return res; From bf8ebde0b1f8dd6de4f4a110e34889053ce72913 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 6 Sep 2022 12:04:08 -0700 Subject: [PATCH 03/61] Bump version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d37ffca5..a3784636 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "umami", - "version": "1.38.0", + "version": "1.39.0-beta.1", "description": "A simple, fast, privacy-focused alternative to Google Analytics.", "author": "Mike Cao ", "license": "MIT", From 52e036964b5d057f851903aa2a4fa0334ef9594d Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 12 Sep 2022 09:55:34 -0700 Subject: [PATCH 04/61] Merge Session/Event/Pageview CH --- .../migrations/01_init/migration.sql | 144 +++++------------- lib/clickhouse.js | 50 +++--- lib/session.js | 75 +++++---- pages/api/collect.js | 12 +- pages/api/website/[id]/metrics.js | 23 ++- pages/api/website/[id]/pageviews.js | 40 +++-- queries/analytics/event/getEvents.js | 2 +- queries/analytics/event/saveEvent.js | 6 +- .../analytics/pageview/getPageviewMetrics.js | 28 ++-- .../analytics/pageview/getPageviewStats.js | 42 ++--- queries/analytics/pageview/getPageviews.js | 5 +- queries/analytics/pageview/savePageView.js | 12 +- queries/analytics/session/createSession.js | 27 ++-- queries/analytics/session/getSessionByUuid.js | 4 +- .../analytics/session/getSessionMetrics.js | 33 ++-- queries/analytics/session/getSessions.js | 4 +- queries/analytics/stats/getActiveVisitors.js | 2 +- queries/analytics/stats/getWebsiteStats.js | 22 +-- 18 files changed, 237 insertions(+), 294 deletions(-) diff --git a/db/clickhouse/migrations/01_init/migration.sql b/db/clickhouse/migrations/01_init/migration.sql index 5a1df775..6bcf899c 100644 --- a/db/clickhouse/migrations/01_init/migration.sql +++ b/db/clickhouse/migrations/01_init/migration.sql @@ -1,118 +1,50 @@ SET allow_experimental_object_type = 1; --- Create Pageview -CREATE TABLE pageview -( - website_id UInt32, - session_uuid UUID, - created_at DateTime('UTC'), - url String, - referrer String -) - engine = MergeTree PRIMARY KEY (session_uuid, created_at) - ORDER BY (session_uuid, created_at) - SETTINGS index_granularity = 8192; - -CREATE TABLE pageview_queue ( - website_id UInt32, - session_uuid UUID, - created_at DateTime('UTC'), - url String, - referrer String -) -ENGINE = Kafka -SETTINGS kafka_broker_list = 'kafka1:19092,kafka2:19093,kafka3:19094', -- input broker list - kafka_topic_list = 'pageview', - kafka_group_name = 'pageview_consumer_group', - kafka_format = 'JSONEachRow', - kafka_max_block_size = 1048576, - kafka_skip_broken_messages = 1; - -CREATE MATERIALIZED VIEW pageview_queue_mv TO pageview AS -SELECT website_id, - session_uuid, - created_at, - url, - referrer -FROM pageview_queue; - --- Create Session -CREATE TABLE session -( - session_uuid UUID, - website_id UInt32, - created_at DateTime('UTC'), - hostname LowCardinality(String), - browser LowCardinality(String), - os LowCardinality(String), - device LowCardinality(String), - screen LowCardinality(String), - language LowCardinality(String), - country LowCardinality(String) -) - engine = MergeTree PRIMARY KEY (session_uuid, created_at) - ORDER BY (session_uuid, created_at) - SETTINGS index_granularity = 8192; - -CREATE TABLE session_queue ( - session_uuid UUID, - website_id UInt32, - created_at DateTime('UTC'), - hostname LowCardinality(String), - browser LowCardinality(String), - os LowCardinality(String), - device LowCardinality(String), - screen LowCardinality(String), - language LowCardinality(String), - country LowCardinality(String) -) -ENGINE = Kafka -SETTINGS kafka_broker_list = 'kafka1:19092,kafka2:19093,kafka3:19094', -- input broker list - kafka_topic_list = 'session', - kafka_group_name = 'session_consumer_group', - kafka_format = 'JSONEachRow', - kafka_max_block_size = 1048576, - kafka_skip_broken_messages = 1; - -CREATE MATERIALIZED VIEW session_queue_mv TO session AS -SELECT session_uuid, - website_id, - created_at, - hostname, - browser, - os, - device, - screen, - language, - country -FROM session_queue; - --- Create event +-- Create Event CREATE TABLE event ( - event_uuid UUID, website_id UInt32, session_uuid UUID, - created_at DateTime('UTC'), + event_uuid Nullable(UUID), + --session + hostname LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + device LowCardinality(String), + screen LowCardinality(String), + language LowCardinality(String), + country LowCardinality(String), + --pageview url String, + referrer String, + --event event_name String, - event_data JSON + event_data JSON, + created_at DateTime('UTC') ) - engine = MergeTree PRIMARY KEY (event_uuid, created_at) - ORDER BY (event_uuid, created_at) + engine = MergeTree + ORDER BY (website_id, session_uuid, created_at) SETTINGS index_granularity = 8192; CREATE TABLE event_queue ( - event_uuid UUID, website_id UInt32, session_uuid UUID, - created_at DateTime('UTC'), + event_uuid Nullable(UUID), url String, + referrer String, + hostname LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + device LowCardinality(String), + screen LowCardinality(String), + language LowCardinality(String), + country LowCardinality(String), event_name String, - event_data String + event_data String, + created_at DateTime('UTC') ) ENGINE = Kafka -SETTINGS kafka_broker_list = 'kafka1:19092,kafka2:19093,kafka3:19094', -- input broker list +SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input broker list kafka_topic_list = 'event', kafka_group_name = 'event_consumer_group', kafka_format = 'JSONEachRow', @@ -120,11 +52,19 @@ SETTINGS kafka_broker_list = 'kafka1:19092,kafka2:19093,kafka3:19094', -- input kafka_skip_broken_messages = 1; CREATE MATERIALIZED VIEW event_queue_mv TO event AS -SELECT event_uuid, - website_id, +SELECT website_id, session_uuid, - created_at, + event_uuid, url, + referrer, + hostname, + browser, + os, + device, + screen, + language, + country, event_name, - event_data -FROM event_queue; + event_data, + created_at +FROM event_queue; \ No newline at end of file diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 519899f2..84be902b 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -62,7 +62,7 @@ function getBetweenDates(field, start_at, end_at) { and ${getDateFormat(end_at)}`; } -function getFilterQuery(table, column, filters = {}, params = []) { +function getFilterQuery(column, filters = {}, params = []) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; @@ -72,48 +72,36 @@ function getFilterQuery(table, column, filters = {}, params = []) { switch (key) { case 'url': - if (table === 'pageview' || table === 'event') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(filter)); - } + arr.push(`and ${key}=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); break; case 'os': case 'browser': case 'device': case 'country': - if (table === 'session') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(filter)); - } + arr.push(`and ${key}=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); break; case 'event_name': - if (table === 'event') { - arr.push(`and ${table}.${key}=$${params.length + 1}`); - params.push(decodeURIComponent(filter)); - } + arr.push(`and ${key}=$${params.length + 1}`); + params.push(decodeURIComponent(filter)); break; case 'referrer': - if (table === 'pageview' || table === 'event') { - arr.push(`and ${table}.referrer like $${params.length + 1}`); - params.push(`%${decodeURIComponent(filter)}%`); - } + arr.push(`and referrer like $${params.length + 1}`); + params.push(`%${decodeURIComponent(filter)}%`); break; case 'domain': - if (table === 'pageview') { - arr.push(`and ${table}.referrer not like $${params.length + 1}`); - arr.push(`and ${table}.referrer not like '/%'`); - params.push(`%://${filter}/%`); - } + arr.push(`and referrer not like $${params.length + 1}`); + arr.push(`and referrer not like '/%'`); + params.push(`%://${filter}/%`); break; case 'query': - if (table === 'pageview') { - arr.push(`and ${table}.url like '%?%'`); - } + arr.push(`and url like '%?%'`); } return arr; @@ -122,7 +110,7 @@ function getFilterQuery(table, column, filters = {}, params = []) { return query.join('\n'); } -function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') { +function parseFilters(column, filters = {}, params = []) { const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } = filters; @@ -135,13 +123,9 @@ function parseFilters(table, column, filters = {}, params = [], sessionKey = 'se sessionFilters, eventFilters, event: { event_name }, - joinSession: - os || browser || device || country - ? `inner join session on ${table}.${sessionKey} = session.${sessionKey}` - : '', - pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params), - sessionQuery: getFilterQuery('session', column, sessionFilters, params), - eventQuery: getFilterQuery('event', column, eventFilters, params), + pageviewQuery: getFilterQuery(column, pageviewFilters, params), + sessionQuery: getFilterQuery(column, sessionFilters, params), + eventQuery: getFilterQuery(column, eventFilters, params), }; } diff --git a/lib/session.js b/lib/session.js index 6e78333b..883726e7 100644 --- a/lib/session.js +++ b/lib/session.js @@ -4,9 +4,12 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import clickhouse from 'lib/clickhouse'; export async function getSession(req) { const { payload } = getJsonBody(req); + const isRedis = redis.client; + const isClickhouse = clickhouse.client; if (!payload) { throw new Error('Invalid request'); @@ -31,11 +34,11 @@ export async function getSession(req) { let websiteId = null; // Check if website exists - if (redis.client) { + if (isRedis) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if redis does not have + // Check database if does not exists in Redis if (!websiteId) { const website = await getWebsiteByUuid(website_uuid); websiteId = website ? website.website_id : null; @@ -46,47 +49,57 @@ export async function getSession(req) { } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); - const session_uuid = uuid(websiteId, hostname, ip, userAgent); let sessionId = null; let session = null; - // Check if session exists - if (redis.client) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if redis does not have - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); + if (!isClickhouse) { + // Check if session exists + if (isRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); sessionId = session ? session.session_id : null; - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } } } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; } return { website_id: websiteId, - session_id: sessionId, - session_uuid, + session, }; } diff --git a/pages/api/collect.js b/pages/api/collect.js index 7fd7ffdf..f253d09e 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -59,7 +59,7 @@ export default async (req, res) => { await useSession(req, res); const { - session: { website_id, session_id, session_uuid }, + session: { website_id, session }, } = req; const { type, payload } = getJsonBody(req); @@ -73,12 +73,11 @@ export default async (req, res) => { const event_uuid = uuid(); if (type === 'pageview') { - await savePageView(website_id, { session_id, session_uuid, url, referrer }); + await savePageView(website_id, { session, url, referrer }); } else if (type === 'event') { await saveEvent(website_id, { + session, event_uuid, - session_id, - session_uuid, url, event_name, event_data, @@ -87,7 +86,10 @@ export default async (req, res) => { return badRequest(res); } - const token = createToken({ website_id, session_id, session_uuid }, secret()); + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); return send(res, token); }; diff --git a/pages/api/website/[id]/metrics.js b/pages/api/website/[id]/metrics.js index 378d8c73..206209c6 100644 --- a/pages/api/website/[id]/metrics.js +++ b/pages/api/website/[id]/metrics.js @@ -48,11 +48,16 @@ export default async (req, res) => { const endDate = new Date(+end_at); if (sessionColumns.includes(type)) { - let data = await getSessionMetrics(websiteId, startDate, endDate, type, { - os, - browser, - device, - country, + let data = await getSessionMetrics(websiteId, { + startDate, + endDate, + field: type, + filters: { + os, + browser, + device, + country, + }, }); if (type === 'language') { @@ -101,7 +106,13 @@ export default async (req, res) => { query: type === 'query' && table !== 'event' ? true : undefined, }; - const data = await getPageviewMetrics(websiteId, startDate, endDate, column, table, filters); + const data = await getPageviewMetrics(websiteId, { + startDate, + endDate, + column, + table, + filters, + }); return ok(res, data); } diff --git a/pages/api/website/[id]/pageviews.js b/pages/api/website/[id]/pageviews.js index 9a713761..4277250f 100644 --- a/pages/api/website/[id]/pageviews.js +++ b/pages/api/website/[id]/pageviews.js @@ -26,20 +26,34 @@ export default async (req, res) => { } const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, startDate, endDate, tz, unit, '*', { - url, - referrer, - os, - browser, - device, - country, + getPageviewStats(websiteId, { + startDate, + endDate, + tz, + unit, + count: '*', + filters: { + url, + referrer, + os, + browser, + device, + country, + }, }), - getPageviewStats(websiteId, startDate, endDate, tz, unit, 'distinct pageview.', { - url, - os, - browser, - device, - country, + getPageviewStats(websiteId, { + startDate, + endDate, + tz, + unit, + count: 'distinct pageview.', + filters: { + url, + os, + browser, + device, + country, + }, }), ]); diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 317a8f2e..41f8da70 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -29,7 +29,7 @@ function clickhouseQuery(websites, start_at) { return rawQuery( `select - event_id, + event_uuid, website_id, session_id, created_at, diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 2c09f05d..3b10f4cb 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -33,17 +33,19 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ async function clickhouseQuery( website_id, - { event_uuid, session_uuid, url, event_name, event_data }, + { session: { country, ...sessionArgs }, event_uuid, url, event_name, event_data }, ) { const { getDateFormat, sendMessage } = kafka; + const params = { event_uuid, website_id, - session_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: event_name?.substring(0, EVENT_NAME_LENGTH), event_data: JSON.stringify(event_data), + ...sessionArgs, + country: country ? country : null, }; await sendMessage(params, 'event'); diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index d34f4c17..4d0f1455 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -9,9 +9,9 @@ export async function getPageviewMetrics(...args) { }); } -async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) { +async function relationalQuery(website_id, { startDate, endDate, column, table, filters = {} }) { const { rawQuery, parseFilters } = prisma; - const params = [website_id, start_at, end_at]; + const params = [website_id, startDate, endDate]; const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( table, column, @@ -34,26 +34,20 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt ); } -async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) { +async function clickhouseQuery(website_id, { startDate, endDate, column, filters = {} }) { const { rawQuery, parseFilters, getBetweenDates } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters( - table, - column, - filters, - params, - 'session_uuid', - ); + const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params); return rawQuery( `select ${column} x, count(*) y - from ${table} - ${joinSession} - where ${table}.website_id= $1 - and ${getBetweenDates(table + '.created_at', start_at, end_at)} - ${pageviewQuery} - ${joinSession && sessionQuery} - ${eventQuery} + from event + where website_id= $1 + ${column !== 'event_name' ? `and event_name = ''` : ''} + and ${getBetweenDates('created_at', startDate, endDate)} + ${pageviewQuery} + ${sessionQuery} + ${eventQuery} group by x order by y desc`, params, diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index da25ab0c..1cf4aba6 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -11,13 +11,15 @@ export async function getPageviewStats(...args) { async function relationalQuery( website_id, - start_at, - end_at, - timezone = 'utc', - unit = 'day', - count = '*', - filters = {}, - sessionKey = 'session_id', + { + start_at, + end_at, + timezone = 'utc', + unit = 'day', + count = '*', + filters = {}, + sessionKey = 'session_id', + }, ) { const { getDateQuery, parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; @@ -44,23 +46,11 @@ async function relationalQuery( async function clickhouseQuery( website_id, - start_at, - end_at, - timezone = 'UTC', - unit = 'day', - count = '*', - filters = {}, - sessionKey = 'session_uuid', + { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} }, ) { const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters( - 'pageview', - null, - filters, - params, - sessionKey, - ); + const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select @@ -69,11 +59,11 @@ async function clickhouseQuery( from (select ${getDateQuery('created_at', unit, timezone)} t, - count(${count !== '*' ? `${count}${sessionKey}` : count}) y - from pageview - ${joinSession} - where pageview.website_id= $1 - and ${getBetweenDates('pageview.created_at', start_at, end_at)} + count(${count !== '*' ? 'session_uuid' : count}) y + from event + where website_id= $1 + + and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} group by t) g diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 8ce704d8..37eaef2d 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -32,8 +32,9 @@ async function clickhouseQuery(websites, start_at) { session_id, created_at, url - from pageview - where website_id in (${websites.join[',']} + from event + where event_name = '' + and website_id in (${websites.join[',']} and created_at >= ${clickhouse.getDateFormat(start_at)})`, ); } diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 3739ff43..826d9275 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -10,7 +10,7 @@ export async function savePageView(...args) { }); } -async function relationalQuery(website_id, { session_id, url, referrer }) { +async function relationalQuery(website_id, { session: { session_id }, url, referrer }) { return prisma.client.pageview.create({ data: { website_id, @@ -21,15 +21,19 @@ async function relationalQuery(website_id, { session_id, url, referrer }) { }); } -async function clickhouseQuery(website_id, { session_uuid, url, referrer }) { +async function clickhouseQuery( + website_id, + { session: { country, ...sessionArgs }, url, referrer }, +) { const { getDateFormat, sendMessage } = kafka; const params = { website_id: website_id, - session_uuid: session_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), referrer: referrer?.substring(0, URL_LENGTH), + ...sessionArgs, + country: country ? country : null, }; - await sendMessage(params, 'pageview'); + await sendMessage(params, 'event'); } diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 4e23dac1..5130b960 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -19,6 +19,14 @@ async function relationalQuery(website_id, data) { }, select: { session_id: true, + session_uuid: true, + hostname: true, + browser: true, + os: true, + screen: true, + language: true, + country: true, + device: true, }, }) .then(async res => { @@ -35,20 +43,21 @@ async function clickhouseQuery( { session_uuid, hostname, browser, os, screen, language, country, device }, ) { const { getDateFormat, sendMessage } = kafka; + const params = { - session_uuid: session_uuid, - website_id: website_id, + session_uuid, + website_id, created_at: getDateFormat(new Date()), - hostname: hostname, - browser: browser, - os: os, - device: device, - screen: screen, - language: language, + hostname, + browser, + os, + device, + screen, + language, country: country ? country : null, }; - await sendMessage(params, 'session'); + await sendMessage(params, 'event'); if (redis.client) { await redis.client.set(`session:${session_uuid}`, 1); diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js index 911848dc..a2b5754d 100644 --- a/queries/analytics/session/getSessionByUuid.js +++ b/queries/analytics/session/getSessionByUuid.js @@ -31,7 +31,7 @@ async function clickhouseQuery(session_uuid) { const params = [session_uuid]; return rawQuery( - `select + `select distinct session_uuid, website_id, created_at, @@ -42,7 +42,7 @@ async function clickhouseQuery(session_uuid) { screen, language, country - from session + from event where session_uuid = $1`, params, ) diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index 36eb7568..796a0682 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -9,15 +9,10 @@ export async function getSessionMetrics(...args) { }); } -async function relationalQuery(website_id, start_at, end_at, field, filters = {}) { +async function relationalQuery(website_id, { startDate, endDate, field, filters = {} }) { const { parseFilters, rawQuery } = prisma; - const params = [website_id, start_at, end_at]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters( - 'pageview', - null, - filters, - params, - ); + const params = [website_id, startDate, endDate]; + const { pageviewQuery, sessionQuery, joinSession } = parseFilters(null, filters, params); return rawQuery( `select ${field} x, count(*) y @@ -37,29 +32,19 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {} ); } -async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) { +async function clickhouseQuery(website_id, { startDate, endDate, field, filters = {} }) { const { parseFilters, getBetweenDates, rawQuery } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters( - 'pageview', - null, - filters, - params, - 'session_uuid', - ); + const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select ${field} x, count(*) y - from session as x - where x.session_uuid in ( - select pageview.session_uuid - from pageview - ${joinSession} - where pageview.website_id=$1 - and ${getBetweenDates('pageview.created_at', start_at, end_at)} + from event as x + where website_id=$1 + and event_name = '' + and ${getBetweenDates('created_at', startDate, endDate)} ${pageviewQuery} ${sessionQuery} - ) group by x order by y desc`, params, diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index 36095d0e..16dd4047 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -32,7 +32,7 @@ async function clickhouseQuery(websites, start_at) { const { rawQuery, getDateFormat } = clickhouse; return rawQuery( - `select + `select distinct session_uuid, website_id, created_at, @@ -43,7 +43,7 @@ async function clickhouseQuery(websites, start_at) { screen, language, country - from session + from event where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'} and created_at >= ${getDateFormat(start_at)}`, ); diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index 023c00a4..efc6a8c2 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -29,7 +29,7 @@ async function clickhouseQuery(website_id) { return rawQuery( `select count(distinct session_uuid) x - from pageview + from event where website_id = $1 and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`, params, diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 5d280563..6e2693a1 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -44,13 +44,7 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; const params = [website_id]; - const { pageviewQuery, sessionQuery, joinSession } = parseFilters( - 'pageview', - null, - filters, - params, - 'session_uuid', - ); + const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select @@ -59,18 +53,18 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" from ( - select pageview.session_uuid, - ${getDateQuery('pageview.created_at', 'day')} time_series, + select session_uuid, + ${getDateQuery('created_at', 'day')} time_series, count(*) c, min(created_at) min_time, max(created_at) max_time - from pageview - ${joinSession} - where pageview.website_id = $1 - and ${getBetweenDates('pageview.created_at', start_at, end_at)} + from event + where event_name = '' + and website_id = $1 + and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} - group by pageview.session_uuid, time_series + group by session_uuid, time_series ) t;`, params, ); From 8e068310b81c88f14272db9b6c1b6ac68c023824 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 12 Sep 2022 10:12:17 -0700 Subject: [PATCH 05/61] remove CH migration --- db/clickhouse/{migrations/01_init/migration.sql => schema.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/clickhouse/{migrations/01_init/migration.sql => schema.sql} (100%) diff --git a/db/clickhouse/migrations/01_init/migration.sql b/db/clickhouse/schema.sql similarity index 100% rename from db/clickhouse/migrations/01_init/migration.sql rename to db/clickhouse/schema.sql From fb6aabaf06bd928c70c4d0ca72a4f1cbdde8c406 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Mon, 12 Sep 2022 14:36:27 -0700 Subject: [PATCH 06/61] Updated language bundles. --- package.json | 2 +- public/intl/messages/mn-MN.json | 18 ++++++++++++------ public/intl/messages/zh-CN.json | 6 ++++++ yarn.lock | 8 ++++---- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index a3784636..9ad9422c 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "maxmind": "^4.3.6", "moment-timezone": "^0.5.35", "next": "^12.2.5", - "next-basics": "^0.7.0", + "next-basics": "^0.12.0", "node-fetch": "^3.2.8", "npm-run-all": "^4.1.5", "prop-types": "^15.7.2", diff --git a/public/intl/messages/mn-MN.json b/public/intl/messages/mn-MN.json index 6430e55e..77b704ac 100644 --- a/public/intl/messages/mn-MN.json +++ b/public/intl/messages/mn-MN.json @@ -244,7 +244,7 @@ "label.none": [ { "type": 0, - "value": "None" + "value": "Байхгүй" } ], "label.owner": [ @@ -397,6 +397,12 @@ "value": "Вебүүд" } ], + "label.yesterday": [ + { + "type": 0, + "value": "Өчигдөр" + } + ], "message.active-users": [ { "type": 0, @@ -456,7 +462,7 @@ "message.confirm-reset": [ { "type": 0, - "value": "Are your sure you want to reset " + "value": "Та " }, { "type": 1, @@ -464,7 +470,7 @@ }, { "type": 0, - "value": "'s statistics?" + "value": "-н тоон үзүүлэлтүүдийг устгахдаа итгэлтэй байна уу?" } ], "message.copied": [ @@ -482,7 +488,7 @@ "message.edit-dashboard": [ { "type": 0, - "value": "Edit dashboard" + "value": "Хянах самбар засах" } ], "message.failure": [ @@ -770,7 +776,7 @@ "metrics.query-parameters": [ { "type": 0, - "value": "Query parameters" + "value": "Query параметр" } ], "metrics.referrers": [ @@ -782,7 +788,7 @@ "metrics.screens": [ { "type": 0, - "value": "Screens" + "value": "Дэлгэц" } ], "metrics.unique-visitors": [ diff --git a/public/intl/messages/zh-CN.json b/public/intl/messages/zh-CN.json index 5773f835..ab279c3e 100644 --- a/public/intl/messages/zh-CN.json +++ b/public/intl/messages/zh-CN.json @@ -397,6 +397,12 @@ "value": "网站" } ], + "label.yesterday": [ + { + "type": 0, + "value": "昨天" + } + ], "message.active-users": [ { "type": 0, diff --git a/yarn.lock b/yarn.lock index f4b49dc1..91c1e9d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4731,10 +4731,10 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next-basics@^0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/next-basics/-/next-basics-0.7.0.tgz#ef2a23dcb3bbfb2170df26b8b5f50c823edc1a2a" - integrity sha512-Z6u8IU77XqadrntFx98CVmfOBcDOiHvemUW753STMte81W9TLflRgOSg1thIg5iwgzQfvCIaU5I1sd06XvvdeA== +next-basics@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/next-basics/-/next-basics-0.12.0.tgz#e946cdb3f32d5d84f67e7c35b6e2e96750763c62" + integrity sha512-3meED1Z9m8QvOsG4bz5OAuyjROdV9Cf75Wq2t+cHiGBh+y8qhTluJduqIXuoY5x0S72/5XvY1mwthW+Og47H9A== dependencies: bcryptjs "^2.4.3" jsonwebtoken "^8.5.1" From c73d07d3294418b4d41492c69fa0f62ca373dfe2 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 13 Sep 2022 09:16:21 -0700 Subject: [PATCH 07/61] clarified var --- lib/session.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/session.js b/lib/session.js index 883726e7..7feafc15 100644 --- a/lib/session.js +++ b/lib/session.js @@ -8,8 +8,8 @@ import clickhouse from 'lib/clickhouse'; export async function getSession(req) { const { payload } = getJsonBody(req); - const isRedis = redis.client; - const isClickhouse = clickhouse.client; + const hasRedis = redis.client; + const hasClickhouse = clickhouse.client; if (!payload) { throw new Error('Invalid request'); @@ -34,7 +34,7 @@ export async function getSession(req) { let websiteId = null; // Check if website exists - if (isRedis) { + if (hasRedis) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } @@ -54,9 +54,9 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!isClickhouse) { + if (!hasClickhouse) { // Check if session exists - if (isRedis) { + if (hasRedis) { sessionId = Number(await redis.client.get(`session:${session_uuid}`)); } From dd54fb3d2c09fb3aa90694eb0943766c2ef35db7 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 13 Sep 2022 09:18:58 -0700 Subject: [PATCH 08/61] combine logic --- lib/clickhouse.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 84be902b..15304214 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -72,18 +72,10 @@ function getFilterQuery(column, filters = {}, params = []) { switch (key) { case 'url': - arr.push(`and ${key}=$${params.length + 1}`); - params.push(decodeURIComponent(filter)); - break; - case 'os': case 'browser': case 'device': case 'country': - arr.push(`and ${key}=$${params.length + 1}`); - params.push(decodeURIComponent(filter)); - break; - case 'event_name': arr.push(`and ${key}=$${params.length + 1}`); params.push(decodeURIComponent(filter)); From d8eb22932b6920f14c6b8d424c8ce5dee577d608 Mon Sep 17 00:00:00 2001 From: AriaieBOY Date: Mon, 19 Sep 2022 19:28:52 +0430 Subject: [PATCH 09/61] Update RealtimeLog.js --- components/metrics/RealtimeLog.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/metrics/RealtimeLog.js b/components/metrics/RealtimeLog.js index b6d3dad7..db53ffd0 100644 --- a/components/metrics/RealtimeLog.js +++ b/components/metrics/RealtimeLog.js @@ -15,6 +15,7 @@ import Visitor from 'assets/visitor.svg'; import Eye from 'assets/eye.svg'; import { stringToColor } from 'lib/format'; import { dateFormat } from 'lib/date'; +import { safeDecodeURI } from 'next-basics'; import styles from './RealtimeLog.module.css'; const TYPE_ALL = 0; @@ -113,7 +114,7 @@ export default function RealtimeLog({ data, websites, websiteId }) { target="_blank" rel="noreferrer noopener" > - {url} + {safeDecodeURI(url)} ); } From e92cb29025970bce67902d950f682b38b858d132 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Dehghani Date: Wed, 21 Sep 2022 17:49:18 +0430 Subject: [PATCH 10/61] open update links in new tab --- components/common/UpdateNotice.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/common/UpdateNotice.js b/components/common/UpdateNotice.js index 7419abe8..54c5fb83 100644 --- a/components/common/UpdateNotice.js +++ b/components/common/UpdateNotice.js @@ -18,7 +18,7 @@ export default function UpdateNotice() { function handleViewClick() { updateCheck(); setDismissed(true); - location.href = releaseUrl || REPO_URL; + open(releaseUrl || REPO_URL, '_blank'); } function handleDismissClick() { From 55218d1dddf0b4ae37f14593d4ef7ab0eecd76fa Mon Sep 17 00:00:00 2001 From: Francis Cao <31608805+franciscao633@users.noreply.github.com> Date: Wed, 21 Sep 2022 11:31:52 -0700 Subject: [PATCH 11/61] Francis/uc 37 secure kafka (#1532) * add ssl encryption to kafka client * fix missing columns in getPageview CH. fix Kafka SSL Pathing --- .gitignore | 3 +++ lib/kafka.js | 10 ++++++++-- queries/analytics/pageview/getPageviews.js | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 54410324..0bad6264 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ yarn-error.log* *.dev.yml +# cert +/lib/cert + diff --git a/lib/kafka.js b/lib/kafka.js index 6b6f315f..4ad119ca 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -8,11 +8,17 @@ const log = debug('umami:kafka'); function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); + const fs = require('fs'); const ssl = username && password ? { - ssl: true, + ssl: { + checkServerIdentity: () => undefined, + ca: [fs.readFileSync('./lib/cert/ca_cert.pem', 'utf-8')], + key: fs.readFileSync('./lib/cert/client_key.pem', 'utf-8'), + cert: fs.readFileSync('./lib/cert/client_cert.pem', 'utf-8'), + }, sasl: { mechanism: 'plain', username, @@ -63,7 +69,7 @@ async function sendMessage(params, topic) { value: JSON.stringify(params), }, ], - acks: 0, + acks: 1, }); } diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 37eaef2d..2bbfff69 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -27,9 +27,8 @@ async function relationalQuery(websites, start_at) { async function clickhouseQuery(websites, start_at) { return clickhouse.rawQuery( `select - view_id, website_id, - session_id, + session_uuid, created_at, url from event From dfac7e1af5980e4fb795c20b483e251edacee5de Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 22 Sep 2022 10:36:23 -0700 Subject: [PATCH 12/61] Updated Kafka loading process. --- lib/kafka.js | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/kafka.js b/lib/kafka.js index 4ad119ca..03833585 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -8,16 +8,15 @@ const log = debug('umami:kafka'); function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); - const fs = require('fs'); const ssl = username && password ? { ssl: { checkServerIdentity: () => undefined, - ca: [fs.readFileSync('./lib/cert/ca_cert.pem', 'utf-8')], - key: fs.readFileSync('./lib/cert/client_key.pem', 'utf-8'), - cert: fs.readFileSync('./lib/cert/client_cert.pem', 'utf-8'), + ca: [process.env.CA_CERT], + key: process.env.CLIENT_KEY, + cert: process.env.CLIENT_CERT, }, sasl: { mechanism: 'plain', @@ -62,6 +61,8 @@ function getDateFormat(date) { } async function sendMessage(params, topic) { + await getKafka(); + await producer.send({ topic, messages: [ @@ -73,21 +74,25 @@ async function sendMessage(params, topic) { }); } +async function getKafka() { + if (!kafka) { + kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); + + if (kafka) { + producer = global[KAFKA_PRODUCER] || (await getProducer()); + } + } + + return kafka; +} + // Initialization let kafka; let producer; -(async () => { - kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); - - if (kafka) { - producer = global[KAFKA_PRODUCER] || (await getProducer()); - } -})(); - export default { client: kafka, - producer: producer, + producer, log, getDateFormat, sendMessage, From 78c34536393428048fd3c2eec3b25fe34b6d403d Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 23 Sep 2022 22:43:51 -0700 Subject: [PATCH 13/61] fix clickhouse calls (#1536) --- .gitignore | 3 -- pages/api/website/[id]/pageviews.js | 8 ++--- pages/api/website/[id]/stats.js | 36 +++++++++++-------- queries/analytics/event/getEventMetrics.js | 3 +- queries/analytics/event/getEvents.js | 7 ++-- .../analytics/pageview/getPageviewMetrics.js | 2 +- .../analytics/pageview/getPageviewStats.js | 6 ++-- queries/analytics/pageview/getPageviews.js | 4 +-- queries/analytics/session/getSessions.js | 2 +- queries/analytics/stats/getWebsiteStats.js | 4 +-- 10 files changed, 41 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 0bad6264..54410324 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,3 @@ yarn-error.log* *.dev.yml -# cert -/lib/cert - diff --git a/pages/api/website/[id]/pageviews.js b/pages/api/website/[id]/pageviews.js index 4277250f..f2e11061 100644 --- a/pages/api/website/[id]/pageviews.js +++ b/pages/api/website/[id]/pageviews.js @@ -27,8 +27,8 @@ export default async (req, res) => { const [pageviews, sessions] = await Promise.all([ getPageviewStats(websiteId, { - startDate, - endDate, + start_at: startDate, + end_at: endDate, tz, unit, count: '*', @@ -42,8 +42,8 @@ export default async (req, res) => { }, }), getPageviewStats(websiteId, { - startDate, - endDate, + start_at: startDate, + end_at: endDate, tz, unit, count: 'distinct pageview.', diff --git a/pages/api/website/[id]/stats.js b/pages/api/website/[id]/stats.js index d652c048..a7741af5 100644 --- a/pages/api/website/[id]/stats.js +++ b/pages/api/website/[id]/stats.js @@ -21,21 +21,29 @@ export default async (req, res) => { const prevStartDate = new Date(+start_at - distance); const prevEndDate = new Date(+end_at - distance); - const metrics = await getWebsiteStats(websiteId, startDate, endDate, { - url, - referrer, - os, - browser, - device, - country, + const metrics = await getWebsiteStats(websiteId, { + start_at: startDate, + end_at: endDate, + filters: { + url, + referrer, + os, + browser, + device, + country, + }, }); - const prevPeriod = await getWebsiteStats(websiteId, prevStartDate, prevEndDate, { - url, - referrer, - os, - browser, - device, - country, + const prevPeriod = await getWebsiteStats(websiteId, { + start_at: prevStartDate, + end_at: prevEndDate, + filters: { + url, + referrer, + os, + browser, + device, + country, + }, }); const stats = Object.keys(metrics[0]).reduce((obj, key) => { diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 4a828e15..c37f1b39 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -52,7 +52,8 @@ async function clickhouseQuery( ${getDateQuery('created_at', unit, timezone)} t, count(*) y from event - where website_id= $1 + where event_name != '' + and website_id= $1 and ${getBetweenDates('created_at', start_at, end_at)} ${getFilterQuery('event', filters, params)} group by x, t diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 41f8da70..441eed64 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -31,12 +31,13 @@ function clickhouseQuery(websites, start_at) { `select event_uuid, website_id, - session_id, + session_uuid, created_at, url, event_name from event - where website_id in (${websites.join[',']} - and created_at >= ${getDateFormat(start_at)})`, + where event_name != '' + and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 4d0f1455..bea3502c 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -43,7 +43,7 @@ async function clickhouseQuery(website_id, { startDate, endDate, column, filters `select ${column} x, count(*) y from event where website_id= $1 - ${column !== 'event_name' ? `and event_name = ''` : ''} + ${column !== 'event_name' ? `and event_name = ''` : `and event_name != ''`} and ${getBetweenDates('created_at', startDate, endDate)} ${pageviewQuery} ${sessionQuery} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 1cf4aba6..4f74cf98 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -59,10 +59,10 @@ async function clickhouseQuery( from (select ${getDateQuery('created_at', unit, timezone)} t, - count(${count !== '*' ? 'session_uuid' : count}) y + count(${count !== '*' ? 'distinct session_uuid' : count}) y from event - where website_id= $1 - + where event_name = '' + and website_id= $1 and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index 2bbfff69..bc909b72 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -33,7 +33,7 @@ async function clickhouseQuery(websites, start_at) { url from event where event_name = '' - and website_id in (${websites.join[',']} - and created_at >= ${clickhouse.getDateFormat(start_at)})`, + and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and created_at >= ${clickhouse.getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js index 16dd4047..7c47ce70 100644 --- a/queries/analytics/session/getSessions.js +++ b/queries/analytics/session/getSessions.js @@ -44,7 +44,7 @@ async function clickhouseQuery(websites, start_at) { language, country from event - where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'} + where ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 6e2693a1..0c3d432a 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -9,7 +9,7 @@ export async function getWebsiteStats(...args) { }); } -async function relationalQuery(website_id, start_at, end_at, filters = {}) { +async function relationalQuery(website_id, { start_at, end_at, filters = {} }) { const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma; const params = [website_id, start_at, end_at]; const { pageviewQuery, sessionQuery, joinSession } = parseFilters( @@ -41,7 +41,7 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) { ); } -async function clickhouseQuery(website_id, start_at, end_at, filters = {}) { +async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; const params = [website_id]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); From b3d7d49448ed8176560c5b5d7d2e7d3fa10ddd1c Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 27 Sep 2022 17:18:16 -0700 Subject: [PATCH 14/61] Fix timezone bug (#1543) --- pages/api/website/[id]/pageviews.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/api/website/[id]/pageviews.js b/pages/api/website/[id]/pageviews.js index f2e11061..f00fffa1 100644 --- a/pages/api/website/[id]/pageviews.js +++ b/pages/api/website/[id]/pageviews.js @@ -29,7 +29,7 @@ export default async (req, res) => { getPageviewStats(websiteId, { start_at: startDate, end_at: endDate, - tz, + timezone: tz, unit, count: '*', filters: { @@ -44,7 +44,7 @@ export default async (req, res) => { getPageviewStats(websiteId, { start_at: startDate, end_at: endDate, - tz, + timezone: tz, unit, count: 'distinct pageview.', filters: { From d4abe51331d53532e18f749e860ae4e955a0a5bd Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Wed, 28 Sep 2022 15:36:41 -0700 Subject: [PATCH 15/61] remove database_type from prisma getdb function --- lib/db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/db.js b/lib/db.js index 58e769b7..e58a18c8 100644 --- a/lib/db.js +++ b/lib/db.js @@ -12,7 +12,7 @@ BigInt.prototype.toJSON = function () { }; export function getDatabaseType(url = process.env.DATABASE_URL) { - const type = process.env.DATABASE_TYPE || (url && url.split(':')[0]); + const type = url && url.split(':')[0]; if (type === 'postgres') { return POSTGRESQL; From 5fb1f08df836c349ce58858b386299572828d45e Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 29 Sep 2022 15:15:11 -0700 Subject: [PATCH 16/61] Split up account update logic. --- components/forms/AccountEditForm.js | 3 +- pages/api/account/[id].js | 59 +++++++++++++++++++++++----- pages/api/account/index.js | 60 ++++++++--------------------- 3 files changed, 66 insertions(+), 56 deletions(-) diff --git a/components/forms/AccountEditForm.js b/components/forms/AccountEditForm.js index 97317aff..86825629 100644 --- a/components/forms/AccountEditForm.js +++ b/components/forms/AccountEditForm.js @@ -33,7 +33,8 @@ export default function AccountEditForm({ values, onSave, onClose }) { const [message, setMessage] = useState(); const handleSubmit = async values => { - const { ok, data } = await post('/account', values); + const { user_id } = values; + const { ok, data } = await post(user_id ? `/account/${user_id}` : '/account', values); if (ok) { onSave(); diff --git a/pages/api/account/[id].js b/pages/api/account/[id].js index 31c4b2dc..e5020f1f 100644 --- a/pages/api/account/[id].js +++ b/pages/api/account/[id].js @@ -1,26 +1,65 @@ -import { getAccountById, deleteAccount } from 'queries'; +import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { getAccountById, deleteAccount, getAccountByUsername, updateAccount } from 'queries'; import { useAuth } from 'lib/middleware'; -import { methodNotAllowed, ok, unauthorized } from 'next-basics'; export default async (req, res) => { await useAuth(req, res); - const { is_admin } = req.auth; + const { is_admin: currentUserIsAdmin, user_id: currentUserId } = req.auth; const { id } = req.query; - const user_id = +id; - - if (!is_admin) { - return unauthorized(res); - } + const userId = +id; if (req.method === 'GET') { - const account = await getAccountById(user_id); + if (userId !== currentUserId && !currentUserIsAdmin) { + return unauthorized(res); + } + + const account = await getAccountById(userId); return ok(res, account); } + if (req.method === 'POST') { + const { username, password, is_admin } = req.body; + + if (userId !== currentUserId && !currentUserIsAdmin) { + return unauthorized(res); + } + + const account = await getAccountById(userId); + + const data = {}; + + if (password) { + data.password = hashPassword(password); + } + + // Only admin can change these fields + if (currentUserIsAdmin) { + data.username = username; + data.is_admin = is_admin; + } + + // Check when username changes + if (data.username && account.username !== data.username) { + const accountByUsername = await getAccountByUsername(username); + + if (accountByUsername) { + return badRequest(res, 'Account already exists'); + } + } + + const updated = await updateAccount(userId, data); + + return ok(res, updated); + } + if (req.method === 'DELETE') { - await deleteAccount(user_id); + if (!currentUserIsAdmin) { + return unauthorized(res); + } + + await deleteAccount(userId); return ok(res); } diff --git a/pages/api/account/index.js b/pages/api/account/index.js index fe9cafe1..236538bc 100644 --- a/pages/api/account/index.js +++ b/pages/api/account/index.js @@ -1,56 +1,26 @@ import { ok, unauthorized, methodNotAllowed, badRequest, hashPassword } from 'next-basics'; -import { getAccountById, getAccountByUsername, updateAccount, createAccount } from 'queries'; +import { getAccountByUsername, createAccount } from 'queries'; import { useAuth } from 'lib/middleware'; export default async (req, res) => { - await useAuth(req, res); - - const { user_id: current_user_id, is_admin: current_user_is_admin } = req.auth; - if (req.method === 'POST') { - const { user_id, username, password, is_admin } = req.body; - - if (user_id) { - const account = await getAccountById(user_id); - - if (account.user_id === current_user_id || current_user_is_admin) { - const data = {}; - - if (password) { - data.password = hashPassword(password); - } - - // Only admin can change these fields - if (current_user_is_admin) { - data.username = username; - data.is_admin = is_admin; - } - - if (data.username && account.username !== data.username) { - const accountByUsername = await getAccountByUsername(username); - - if (accountByUsername) { - return badRequest(res, 'Account already exists'); - } - } - - const updated = await updateAccount(user_id, data); - - return ok(res, updated); - } + await useAuth(req, res); + if (!req.auth.is_admin) { return unauthorized(res); - } else { - const accountByUsername = await getAccountByUsername(username); - - if (accountByUsername) { - return badRequest(res, 'Account already exists'); - } - - const created = await createAccount({ username, password: hashPassword(password) }); - - return ok(res, created); } + + const { username, password } = req.body; + + const accountByUsername = await getAccountByUsername(username); + + if (accountByUsername) { + return badRequest(res, 'Account already exists'); + } + + const created = await createAccount({ username, password: hashPassword(password) }); + + return ok(res, created); } return methodNotAllowed(res); From adb0a0600636703ad3bff2b3cfc72e12b2ebbd66 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Fri, 30 Sep 2022 22:27:47 -0700 Subject: [PATCH 17/61] add uuid to account --- .../migrations/04_account_uuid/migration.sql | 12 ++++++++++++ db/postgresql/schema.prisma | 15 ++++++++------- pages/api/account/index.js | 7 ++++++- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 db/postgresql/migrations/04_account_uuid/migration.sql diff --git a/db/postgresql/migrations/04_account_uuid/migration.sql b/db/postgresql/migrations/04_account_uuid/migration.sql new file mode 100644 index 00000000..f95718d4 --- /dev/null +++ b/db/postgresql/migrations/04_account_uuid/migration.sql @@ -0,0 +1,12 @@ + +-- AlterTable +ALTER TABLE "account" ADD COLUMN "account_uuid" UUID NULL; + +-- Backfill UUID +UPDATE "account" SET account_uuid = gen_random_uuid(); + +-- AlterTable +ALTER TABLE "account" ALTER COLUMN "account_uuid" SET NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX "account_account_uuid_key" ON "account"("account_uuid"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index a76a3da4..d1d346de 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -8,13 +8,14 @@ datasource db { } model account { - user_id Int @id @default(autoincrement()) - username String @unique @db.VarChar(255) - password String @db.VarChar(60) - is_admin Boolean @default(false) - created_at DateTime? @default(now()) @db.Timestamptz(6) - updated_at DateTime? @default(now()) @db.Timestamptz(6) - website website[] + user_id Int @id @default(autoincrement()) + username String @unique @db.VarChar(255) + password String @db.VarChar(60) + is_admin Boolean @default(false) + created_at DateTime? @default(now()) @db.Timestamptz(6) + updated_at DateTime? @default(now()) @db.Timestamptz(6) + account_uuid String @unique @db.Uuid + website website[] } model event { diff --git a/pages/api/account/index.js b/pages/api/account/index.js index fe9cafe1..a3fc6bc1 100644 --- a/pages/api/account/index.js +++ b/pages/api/account/index.js @@ -1,6 +1,7 @@ import { ok, unauthorized, methodNotAllowed, badRequest, hashPassword } from 'next-basics'; import { getAccountById, getAccountByUsername, updateAccount, createAccount } from 'queries'; import { useAuth } from 'lib/middleware'; +import { uuid } from 'lib/crypto'; export default async (req, res) => { await useAuth(req, res); @@ -47,7 +48,11 @@ export default async (req, res) => { return badRequest(res, 'Account already exists'); } - const created = await createAccount({ username, password: hashPassword(password) }); + const created = await createAccount({ + username, + password: hashPassword(password), + account_uuid: uuid(), + }); return ok(res, created); } From 6de634f0b484fb1fa5e7441464b93cb64910f86d Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Sun, 2 Oct 2022 20:42:32 -0700 Subject: [PATCH 18/61] add mysql changes --- db/mysql/migrations/04_account_uuid/migration.sql | 11 +++++++++++ db/mysql/schema.prisma | 15 ++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 db/mysql/migrations/04_account_uuid/migration.sql diff --git a/db/mysql/migrations/04_account_uuid/migration.sql b/db/mysql/migrations/04_account_uuid/migration.sql new file mode 100644 index 00000000..7b7b5dea --- /dev/null +++ b/db/mysql/migrations/04_account_uuid/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE `account` ADD COLUMN `account_uuid` VARCHAR(36); + +-- Backfill UUID +UPDATE `account` SET account_uuid=(SELECT uuid()); + +-- AlterTable +ALTER TABLE `account` MODIFY `account_uuid` VARCHAR(36) NOT NULL; + +-- CreateIndex +CREATE UNIQUE INDEX `account_account_uuid_key` ON `account`(`account_uuid`); diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 9ad2620c..bfc2c20b 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -8,13 +8,14 @@ datasource db { } model account { - user_id Int @id @default(autoincrement()) @db.UnsignedInt - username String @unique() @db.VarChar(255) - password String @db.VarChar(60) - is_admin Boolean @default(false) - created_at DateTime? @default(now()) @db.Timestamp(0) - updated_at DateTime? @default(now()) @db.Timestamp(0) - website website[] + user_id Int @id @default(autoincrement()) @db.UnsignedInt + username String @unique() @db.VarChar(255) + password String @db.VarChar(60) + is_admin Boolean @default(false) + created_at DateTime? @default(now()) @db.Timestamp(0) + updated_at DateTime? @default(now()) @db.Timestamp(0) + account_uuid String @unique() @db.VarChar(36) + website website[] } model event { From a2ce1d833dfb377aa949b47e9045be36ac2c4235 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 3 Oct 2022 11:14:19 -0700 Subject: [PATCH 19/61] update check-db --- scripts/check-db.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/check-db.js b/scripts/check-db.js index 4de18a89..d5cc2a64 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ require('dotenv').config(); const { PrismaClient } = require('@prisma/client'); const chalk = require('chalk'); @@ -39,7 +40,7 @@ async function checkConnection() { async function checkTables() { try { - await prisma.account.findFirst(); + await prisma.$queryRaw`select * from account limit 1`; success('Database tables found.'); } catch (e) { From f3135ee8673c5e0e1eb88f6abd446851ef78c96e Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Tue, 4 Oct 2022 20:16:09 -0700 Subject: [PATCH 20/61] Get account by uuid for cloud operations. --- pages/api/accounts/index.js | 10 +-- pages/api/websites/index.js | 14 +++-- queries/admin/account/getAccount.js | 7 +++ queries/index.js | 94 ++++++++++------------------- 4 files changed, 50 insertions(+), 75 deletions(-) create mode 100644 queries/admin/account/getAccount.js diff --git a/pages/api/accounts/index.js b/pages/api/accounts/index.js index acaf1451..4c2b8b20 100644 --- a/pages/api/accounts/index.js +++ b/pages/api/accounts/index.js @@ -19,13 +19,7 @@ export default async (req, res) => { } if (req.method === 'POST') { - await useAuth(req, res); - - if (!req.auth.is_admin) { - return unauthorized(res); - } - - const { username, password } = req.body; + const { username, password, account_uuid } = req.body; const accountByUsername = await getAccountByUsername(username); @@ -36,7 +30,7 @@ export default async (req, res) => { const created = await createAccount({ username, password: hashPassword(password), - account_uuid: uuid(), + account_uuid: account_uuid || uuid(), }); return ok(res, created); diff --git a/pages/api/websites/index.js b/pages/api/websites/index.js index 108a4a8a..65a078d1 100644 --- a/pages/api/websites/index.js +++ b/pages/api/websites/index.js @@ -1,4 +1,4 @@ -import { createWebsite, getAllWebsites, getUserWebsites } from 'queries'; +import { createWebsite, getAccount, getAllWebsites, getUserWebsites } from 'queries'; import { ok, methodNotAllowed, unauthorized, getRandomChars } from 'next-basics'; import { useAuth } from 'lib/middleware'; import { uuid } from 'lib/crypto'; @@ -6,9 +6,15 @@ import { uuid } from 'lib/crypto'; export default async (req, res) => { await useAuth(req, res); - const { user_id: current_user_id, is_admin } = req.auth; + const { user_id: current_user_id, is_admin, account_uuid } = req.auth; const { user_id, include_all } = req.query; - const userId = +user_id; + let account; + + if (account_uuid) { + account = await getAccount({ account_uuid }); + } + + const userId = account ? account.user_id : +user_id; if (req.method === 'GET') { if (userId && userId !== current_user_id && !is_admin) { @@ -29,7 +35,7 @@ export default async (req, res) => { const { is_admin: currentUserIsAdmin, user_id: currentUserId } = req.auth; const { name, domain, owner, enable_share_url } = req.body; - const website_owner = +owner; + const website_owner = account ? account.user_id : +owner; if (website_owner !== currentUserId && !currentUserIsAdmin) { return unauthorized(res); diff --git a/queries/admin/account/getAccount.js b/queries/admin/account/getAccount.js new file mode 100644 index 00000000..c414f56d --- /dev/null +++ b/queries/admin/account/getAccount.js @@ -0,0 +1,7 @@ +import prisma from 'lib/prisma'; + +export async function getAccount(where) { + return prisma.client.account.findUnique({ + where, + }); +} diff --git a/queries/index.js b/queries/index.js index 517307f6..35d79215 100644 --- a/queries/index.js +++ b/queries/index.js @@ -1,63 +1,31 @@ -import { createAccount } from './admin/account/createAccount'; -import { deleteAccount } from './admin/account/deleteAccount'; -import { getAccountById } from './admin/account/getAccountById'; -import { getAccountByUsername } from './admin/account/getAccountByUsername'; -import { getAccounts } from './admin/account/getAccounts'; -import { updateAccount } from './admin/account/updateAccount'; -import { createWebsite } from './admin/website/createWebsite'; -import { deleteWebsite } from './admin/website/deleteWebsite'; -import { getAllWebsites } from './admin/website/getAllWebsites'; -import { getUserWebsites } from './admin/website/getUserWebsites'; -import { getWebsiteById } from './admin/website/getWebsiteById'; -import { getWebsiteByShareId } from './admin/website/getWebsiteByShareId'; -import { getWebsiteByUuid } from './admin/website/getWebsiteByUuid'; -import { resetWebsite } from './admin/website/resetWebsite'; -import { updateWebsite } from './admin/website/updateWebsite'; -import { getEventMetrics } from './analytics/event/getEventMetrics'; -import { getEvents } from './analytics/event/getEvents'; -import { saveEvent } from './analytics/event/saveEvent'; -import { getPageviewMetrics } from './analytics/pageview/getPageviewMetrics'; -import { getPageviewParams } from './analytics/pageview/getPageviewParams'; -import { getPageviews } from './analytics/pageview/getPageviews'; -import { getPageviewStats } from './analytics/pageview/getPageviewStats'; -import { savePageView } from './analytics/pageview/savePageView'; -import { createSession } from './analytics/session/createSession'; -import { getSessionByUuid } from './analytics/session/getSessionByUuid'; -import { getSessionMetrics } from './analytics/session/getSessionMetrics'; -import { getSessions } from './analytics/session/getSessions'; -import { getActiveVisitors } from './analytics/stats/getActiveVisitors'; -import { getRealtimeData } from './analytics/stats/getRealtimeData'; -import { getWebsiteStats } from './analytics/stats/getWebsiteStats'; - -export { - createWebsite, - deleteWebsite, - getAllWebsites, - getUserWebsites, - getWebsiteById, - getWebsiteByShareId, - getWebsiteByUuid, - resetWebsite, - updateWebsite, - createAccount, - deleteAccount, - getAccountById, - getAccountByUsername, - getAccounts, - updateAccount, - getEventMetrics, - getEvents, - saveEvent, - getPageviewMetrics, - getPageviewParams, - getPageviews, - getPageviewStats, - savePageView, - createSession, - getSessionByUuid, - getSessionMetrics, - getSessions, - getActiveVisitors, - getRealtimeData, - getWebsiteStats, -}; +export * from './admin/account/createAccount'; +export * from './admin/account/deleteAccount'; +export * from './admin/account/getAccount'; +export * from './admin/account/getAccountById'; +export * from './admin/account/getAccountByUsername'; +export * from './admin/account/getAccounts'; +export * from './admin/account/updateAccount'; +export * from './admin/website/createWebsite'; +export * from './admin/website/deleteWebsite'; +export * from './admin/website/getAllWebsites'; +export * from './admin/website/getUserWebsites'; +export * from './admin/website/getWebsiteById'; +export * from './admin/website/getWebsiteByShareId'; +export * from './admin/website/getWebsiteByUuid'; +export * from './admin/website/resetWebsite'; +export * from './admin/website/updateWebsite'; +export * from './analytics/event/getEventMetrics'; +export * from './analytics/event/getEvents'; +export * from './analytics/event/saveEvent'; +export * from './analytics/pageview/getPageviewMetrics'; +export * from './analytics/pageview/getPageviewParams'; +export * from './analytics/pageview/getPageviews'; +export * from './analytics/pageview/getPageviewStats'; +export * from './analytics/pageview/savePageView'; +export * from './analytics/session/createSession'; +export * from './analytics/session/getSessionByUuid'; +export * from './analytics/session/getSessionMetrics'; +export * from './analytics/session/getSessions'; +export * from './analytics/stats/getActiveVisitors'; +export * from './analytics/stats/getRealtimeData'; +export * from './analytics/stats/getWebsiteStats'; From e8a30e7251fdb6c0819886d21bd75bc109912cf6 Mon Sep 17 00:00:00 2001 From: Sergio Mensing <> Date: Wed, 5 Oct 2022 14:44:49 +0200 Subject: [PATCH 21/61] use slice instead of substring for desired result --- lib/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/format.js b/lib/format.js index a85c3ef7..e138831d 100644 --- a/lib/format.js +++ b/lib/format.js @@ -74,7 +74,7 @@ export function stringToColor(str) { let color = '#'; for (let i = 0; i < 3; i++) { let value = (hash >> (i * 8)) & 0xff; - color += ('00' + value.toString(16)).substring(-2); + color += ('00' + value.toString(16)).slice(-2); } return color; } From 0387cf0da01ef3abe02ec939835cc22d05087dd6 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 12:49:49 -0700 Subject: [PATCH 22/61] remove CH import --- lib/session.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/session.js b/lib/session.js index 7feafc15..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -4,12 +4,11 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; -import clickhouse from 'lib/clickhouse'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = redis.client; - const hasClickhouse = clickhouse.client; + const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); From 86bd9a7793adb53bc2b0252a53e4339cdc7a4442 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 23/61] remove postgres collect --- lib/session.js | 61 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -53,49 +52,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, From 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 24/61] break CH fetch --- lib/session.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,7 +2,6 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -36,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 25/61] reject --- lib/session.js | 74 ++++++++++++++++++++++++---------- pages/api/collect.js | 94 +------------------------------------------- 2 files changed, 55 insertions(+), 113 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,12 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -35,16 +37,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,95 +1,5 @@ -const { Resolver } = require('dns').promises; -import isbot from 'isbot'; -import ipaddr from 'ipaddr.js'; -import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; -import { savePageView, saveEvent } from 'queries'; -import { useCors, useSession } from 'lib/middleware'; -import { getJsonBody, getIpAddress } from 'lib/request'; -import { secret, uuid } from 'lib/crypto'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - const { type, payload } = getJsonBody(req); - - let { url, referrer, event_name, event_data } = payload; - - if (process.env.REMOVE_TRAILING_SLASH) { - url = url.replace(/\/$/, ''); - } - - const event_uuid = uuid(); - - if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); - } else if (type === 'event') { - await saveEvent(website_id, { - session, - event_uuid, - url, - event_name, - event_data, - }); - } else { - return badRequest(res); - } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From 75cba808e3e1edacc2ba105210bf4eff846bab03 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 26/61] do not collect --- pages/api/collect.js | 93 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..d071775b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,94 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import ipaddr from 'ipaddr.js'; +import isbot from 'isbot'; +import { secret } from 'lib/crypto'; +import { useCors, useSession } from 'lib/middleware'; +import { getIpAddress } from 'lib/request'; +import { createToken, forbidden, send, unauthorized } from 'next-basics'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + // const { type, payload } = getJsonBody(req); + + // let { url, referrer, event_name, event_data } = payload; + + // if (process.env.REMOVE_TRAILING_SLASH) { + // url = url.replace(/\/$/, ''); + // } + + // const event_uuid = uuid(); + + // if (type === 'pageview' && 1 === 0) { + // await savePageView(website_id, { session, url, referrer }); + // } else if (type === 'event' && 1 === 0) { + // await saveEvent(website_id, { + // session, + // event_uuid, + // url, + // event_name, + // event_data, + // }); + // } else { + // return badRequest(res); + // } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 76c0057d2ea885850feec394f9a87ba96ec1e503 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 27/61] Revert "do not collect" This reverts commit 75cba808e3e1edacc2ba105210bf4eff846bab03. --- pages/api/collect.js | 93 +------------------------------------------- 1 file changed, 2 insertions(+), 91 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index d071775b..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,94 +1,5 @@ -const { Resolver } = require('dns').promises; -import ipaddr from 'ipaddr.js'; -import isbot from 'isbot'; -import { secret } from 'lib/crypto'; -import { useCors, useSession } from 'lib/middleware'; -import { getIpAddress } from 'lib/request'; -import { createToken, forbidden, send, unauthorized } from 'next-basics'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - // const { type, payload } = getJsonBody(req); - - // let { url, referrer, event_name, event_data } = payload; - - // if (process.env.REMOVE_TRAILING_SLASH) { - // url = url.replace(/\/$/, ''); - // } - - // const event_uuid = uuid(); - - // if (type === 'pageview' && 1 === 0) { - // await savePageView(website_id, { session, url, referrer }); - // } else if (type === 'event' && 1 === 0) { - // await saveEvent(website_id, { - // session, - // event_uuid, - // url, - // event_name, - // event_data, - // }); - // } else { - // return badRequest(res); - // } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From f53ea06c231557b66c4809181c71d4d3d01c4820 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 28/61] Revert "reject" This reverts commit 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5. --- lib/session.js | 74 ++++++++++------------------------ pages/api/collect.js | 94 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 55 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,12 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -37,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,49 +53,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..f253d09e 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,95 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import isbot from 'isbot'; +import ipaddr from 'ipaddr.js'; +import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; +import { savePageView, saveEvent } from 'queries'; +import { useCors, useSession } from 'lib/middleware'; +import { getJsonBody, getIpAddress } from 'lib/request'; +import { secret, uuid } from 'lib/crypto'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + const { type, payload } = getJsonBody(req); + + let { url, referrer, event_name, event_data } = payload; + + if (process.env.REMOVE_TRAILING_SLASH) { + url = url.replace(/\/$/, ''); + } + + const event_uuid = uuid(); + + if (type === 'pageview') { + await savePageView(website_id, { session, url, referrer }); + } else if (type === 'event') { + await saveEvent(website_id, { + session, + event_uuid, + url, + event_name, + event_data, + }); + } else { + return badRequest(res); + } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 0e10470c0286d136c5b4a0235f1f36de209d02ba Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 29/61] Revert "break CH fetch" This reverts commit 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9. --- lib/session.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,6 +2,7 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -35,16 +36,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 6bb61d7a53fe11a2ab03161e1b0513792ca3095a Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 30/61] Revert "remove postgres collect" This reverts commit 86bd9a7793adb53bc2b0252a53e4339cdc7a4442. --- lib/session.js | 61 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,13 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -52,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, From d8c440453cfcda3012e35a5d0f02110f9da3c1bd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 14:18:27 -0700 Subject: [PATCH 31/61] Added getWebsite method. --- lib/auth.js | 8 ++++---- pages/api/websites/[id]/index.js | 6 ++++-- queries/admin/website/getWebsite.js | 7 +++++++ queries/index.js | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 queries/admin/website/getWebsite.js diff --git a/lib/auth.js b/lib/auth.js index b50a923f..b98fb923 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,6 +1,7 @@ +import { validate } from 'uuid'; import { parseSecureToken, parseToken, getItem } from 'next-basics'; import { AUTH_TOKEN, SHARE_TOKEN_HEADER } from './constants'; -import { getWebsiteById } from 'queries'; +import { getWebsite } from 'queries'; import { secret } from './crypto'; export async function getAuthToken(req) { @@ -38,13 +39,12 @@ export async function isValidToken(token, validation) { export async function allowQuery(req, skipToken) { const { id } = req.query; const token = req.headers[SHARE_TOKEN_HEADER]; - const websiteId = +id; - const website = await getWebsiteById(websiteId); + const website = await getWebsite(validate(id) ? { website_uuid: id } : { website_id: +id }); if (website) { if (token && token !== 'undefined' && !skipToken) { - return isValidToken(token, { website_id: websiteId }); + return isValidToken(token, { website_id: website.website_id }); } const authToken = await getAuthToken(req); diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index bc9cb17f..30592213 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,12 +1,14 @@ import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; -import { deleteWebsite, getWebsiteById, updateWebsite } from 'queries'; +import { deleteWebsite, getWebsite, getWebsiteById, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; +import { validate } from 'uuid'; export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const where = validate(id) ? { website_uuid: id } : { website_id: +id }; if (req.method === 'GET') { await useCors(req, res); @@ -15,7 +17,7 @@ export default async (req, res) => { return unauthorized(res); } - const website = await getWebsiteById(websiteId); + const website = await getWebsite(where); return ok(res, website); } diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.js new file mode 100644 index 00000000..83c3e83a --- /dev/null +++ b/queries/admin/website/getWebsite.js @@ -0,0 +1,7 @@ +import prisma from 'lib/prisma'; + +export async function getWebsite(where) { + return prisma.client.website.findUnique({ + where, + }); +} diff --git a/queries/index.js b/queries/index.js index 35d79215..d6b4093a 100644 --- a/queries/index.js +++ b/queries/index.js @@ -9,6 +9,7 @@ export * from './admin/website/createWebsite'; export * from './admin/website/deleteWebsite'; export * from './admin/website/getAllWebsites'; export * from './admin/website/getUserWebsites'; +export * from './admin/website/getWebsite'; export * from './admin/website/getWebsiteById'; export * from './admin/website/getWebsiteByShareId'; export * from './admin/website/getWebsiteByUuid'; From e442617421c85c81b6fa9f53e319b65aa2f06fc4 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 15:00:16 -0700 Subject: [PATCH 32/61] Add connect methods to libraries. --- lib/clickhouse.js | 16 +++++++++-- lib/kafka.js | 14 ++++++---- lib/redis.js | 38 +++++++++++++++++++------- lib/session.js | 17 ++++++------ queries/admin/account/deleteAccount.js | 2 +- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 15304214..9913c0be 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -14,6 +14,9 @@ export const CLICKHOUSE_DATE_FORMATS = { const log = debug('umami:clickhouse'); +let clickhouse; +const enabled = Boolean(process.env.CLICKHOUSE_URL); + function getClient() { const { hostname, @@ -144,6 +147,8 @@ async function rawQuery(query, params = []) { log(formattedQuery); } + await connect(); + return clickhouse.query(formattedQuery).toPromise(); } @@ -159,12 +164,19 @@ async function findFirst(data) { return data[0] ?? null; } -// Initialization -const clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); +async function connect() { + if (!clickhouse) { + clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); + } + + return clickhouse; +} export default { + enabled, client: clickhouse, log, + connect, getDateStringQuery, getDateQuery, getDateFormat, diff --git a/lib/kafka.js b/lib/kafka.js index 03833585..782f2803 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -5,6 +5,10 @@ import { KAFKA, KAFKA_PRODUCER } from 'lib/db'; const log = debug('umami:kafka'); +let kafka; +let producer; +const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER); + function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); @@ -61,7 +65,7 @@ function getDateFormat(date) { } async function sendMessage(params, topic) { - await getKafka(); + await connect(); await producer.send({ topic, @@ -74,7 +78,7 @@ async function sendMessage(params, topic) { }); } -async function getKafka() { +async function connect() { if (!kafka) { kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); @@ -86,14 +90,12 @@ async function getKafka() { return kafka; } -// Initialization -let kafka; -let producer; - export default { + enabled, client: kafka, producer, log, + connect, getDateFormat, sendMessage, }; diff --git a/lib/redis.js b/lib/redis.js index 63b513e6..15ac27d9 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -8,6 +8,9 @@ const log = debug('umami:redis'); const INITIALIZED = 'redis:initialized'; export const DELETED = 'deleted'; +let redis; +const enabled = Boolean(process.env.REDIS_URL); + function getClient() { if (!process.env.REDIS_URL) { return null; @@ -40,26 +43,41 @@ async function stageData() { return { key: `website:${a.website_uuid}`, value: Number(a.website_id) }; }); - await addRedis(sessionUuids); - await addRedis(websiteIds); + await addSet(sessionUuids); + await addSet(websiteIds); await redis.set(INITIALIZED, 1); } -async function addRedis(ids) { +async function addSet(ids) { for (let i = 0; i < ids.length; i++) { const { key, value } = ids[i]; await redis.set(key, value); } } -// Initialization -const redis = process.env.REDIS_URL && (global[REDIS] || getClient()); +async function get(key) { + await connect(); -(async () => { - if (redis && !(await redis.get(INITIALIZED))) { - await stageData(); + return redis.get(key); +} + +async function set(key, value) { + await connect(); + + return redis.set(key, value); +} + +async function connect() { + if (!redis) { + process.env.REDIS_URL && (global[REDIS] || getClient()); + + if (!(await redis.get(INITIALIZED))) { + await stageData(); + } } -})(); -export default { client: redis, stageData, log }; + return redis; +} + +export default { enabled, client: redis, log, connect, get, set, stageData }; diff --git a/lib/session.js b/lib/session.js index d635f75a..9e95cb11 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ import { parseToken } from 'next-basics'; import { validate } from 'uuid'; -import { uuid } from 'lib/crypto'; +import { secret, uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; +import clickhouse from 'lib/clickhouse'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -17,7 +16,7 @@ export async function getSession(req) { const cache = req.headers['x-umami-cache']; if (cache) { - const result = await parseToken(cache); + const result = await parseToken(cache, secret()); if (result) { return result; @@ -33,8 +32,8 @@ export async function getSession(req) { let websiteId = null; // Check if website exists - if (hasRedis) { - websiteId = Number(await redis.client.get(`website:${website_uuid}`)); + if (redis.enabled) { + websiteId = Number(await redis.get(`website:${website_uuid}`)); } // Check database if does not exists in Redis @@ -53,10 +52,10 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { + if (!clickhouse.enabled) { // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + if (redis.enabled) { + sessionId = Number(await redis.get(`session:${session_uuid}`)); } // Check database if does not exists in Redis diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js index 4e204e73..dfe44889 100644 --- a/queries/admin/account/deleteAccount.js +++ b/queries/admin/account/deleteAccount.js @@ -41,7 +41,7 @@ export async function deleteAccount(user_id) { .then(async res => { if (redis.client) { for (let i = 0; i < websiteUuids.length; i++) { - await redis.client.set(`website:${websiteUuids[i]}`, DELETED); + await redis.set(`website:${websiteUuids[i]}`, DELETED); } } From dcf16e1411e80bba2abfc25695d1c0585fa8d45e Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 16:41:37 -0700 Subject: [PATCH 33/61] fix redis init --- lib/redis.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/redis.js b/lib/redis.js index 15ac27d9..01a9e3c6 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -70,11 +70,7 @@ async function set(key, value) { async function connect() { if (!redis) { - process.env.REDIS_URL && (global[REDIS] || getClient()); - - if (!(await redis.get(INITIALIZED))) { - await stageData(); - } + redis = process.env.REDIS_URL && (global[REDIS] || getClient()); } return redis; From 2428314f584e5e21bf9764400bab6b3608f1d4ed Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 7 Oct 2022 17:13:03 -0700 Subject: [PATCH 34/61] Updated save method for websites. --- components/forms/WebsiteEditForm.js | 7 ++++-- pages/api/websites/[id]/index.js | 33 +++++++++++++++----------- queries/admin/website/updateWebsite.js | 6 ++--- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/components/forms/WebsiteEditForm.js b/components/forms/WebsiteEditForm.js index 21daeeaf..337e2d96 100644 --- a/components/forms/WebsiteEditForm.js +++ b/components/forms/WebsiteEditForm.js @@ -14,6 +14,7 @@ import useApi from 'hooks/useApi'; import useFetch from 'hooks/useFetch'; import useUser from 'hooks/useUser'; import styles from './WebsiteEditForm.module.css'; +import { getRandomChars } from 'next-basics'; const initialValues = { name: '', @@ -78,7 +79,10 @@ export default function WebsiteEditForm({ values, onSave, onClose }) { const [message, setMessage] = useState(); const handleSubmit = async values => { - const { website_id } = values; + const { website_id, enable_share_url, share_id } = values; + if (enable_share_url) { + values.share_id = share_id || getRandomChars(8); + } const { ok, data } = await post(website_id ? `/websites/${website_id}` : '/websites', values); if (ok) { @@ -137,7 +141,6 @@ export default function WebsiteEditForm({ values, onSave, onClose }) { defaultMessage="Enable share URL" /> } - value={null} /> )} diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index 30592213..8234a5f2 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,5 +1,5 @@ -import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; -import { deleteWebsite, getWebsite, getWebsiteById, updateWebsite } from 'queries'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { deleteWebsite, getAccount, getWebsite, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; import { validate } from 'uuid'; @@ -25,24 +25,29 @@ export default async (req, res) => { if (req.method === 'POST') { await useAuth(req, res); - const { is_admin: currentUserIsAdmin, user_id: currentUserId } = req.auth; - const { name, domain, owner, enable_share_url } = req.body; + const { is_admin: currentUserIsAdmin, user_id: currentUserId, account_uuid } = req.auth; + const { name, domain, owner, share_id } = req.body; + let account; - const website = await getWebsiteById(websiteId); + if (account_uuid) { + account = await getAccount({ account_uuid }); + } + + const website = await getWebsite(where); if (website.user_id !== currentUserId && !currentUserIsAdmin) { return unauthorized(res); } - let { share_id } = website; - - if (enable_share_url) { - share_id = share_id ? share_id : getRandomChars(8); - } else { - share_id = null; - } - - await updateWebsite(websiteId, { name, domain, share_id, user_id: +owner }); + await updateWebsite( + { + name, + domain, + share_id: share_id || null, + user_id: account ? account.id : +owner, + }, + where, + ); return ok(res); } diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.js index 54f01f66..1a5079a9 100644 --- a/queries/admin/website/updateWebsite.js +++ b/queries/admin/website/updateWebsite.js @@ -1,10 +1,8 @@ import prisma from 'lib/prisma'; -export async function updateWebsite(website_id, data) { +export async function updateWebsite(data, where) { return prisma.client.website.update({ - where: { - website_id, - }, + where, data, }); } From 6f6cbaa09805af605520c9a0b499aab2f1c0a57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=B2=97?= Date: Sat, 8 Oct 2022 13:46:50 +0800 Subject: [PATCH 35/61] Update getWebsiteByUuid.js if redis website set from getWebsiteByUuid,the value is 1 forever --- queries/admin/website/getWebsiteByUuid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.js index 900e4539..1b1e04ca 100644 --- a/queries/admin/website/getWebsiteByUuid.js +++ b/queries/admin/website/getWebsiteByUuid.js @@ -10,7 +10,7 @@ export async function getWebsiteByUuid(website_uuid) { }) .then(async res => { if (redis.client && res) { - await redis.client.set(`website:${res.website_uuid}`, 1); + await redis.client.set(`website:${res.website_uuid}`, res.website_id); } return res; From 96add409b69f76a1a910199143b87957017107f1 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Sat, 8 Oct 2022 16:12:33 -0700 Subject: [PATCH 36/61] update api to new CH columns --- db/clickhouse/schema.sql | 18 +++++++++--------- lib/clickhouse.js | 5 +++++ lib/session.js | 3 ++- pages/api/collect.js | 13 +++++++++---- pages/api/websites/[id]/active.js | 3 ++- pages/api/websites/[id]/events.js | 3 ++- pages/api/websites/[id]/metrics.js | 3 ++- pages/api/websites/[id]/pageviews.js | 5 +++-- pages/api/websites/[id]/stats.js | 5 +++-- queries/analytics/event/getEventMetrics.js | 4 ++-- queries/analytics/event/getEvents.js | 12 ++++++++---- queries/analytics/event/saveEvent.js | 4 ++-- .../analytics/pageview/getPageviewMetrics.js | 4 ++-- queries/analytics/pageview/getPageviewStats.js | 6 +++--- queries/analytics/pageview/getPageviews.js | 10 ++++++++-- queries/analytics/pageview/savePageView.js | 4 ++-- queries/analytics/session/createSession.js | 4 ++-- queries/analytics/stats/getActiveVisitors.js | 6 +++--- queries/analytics/stats/getWebsiteStats.js | 10 +++++----- 19 files changed, 74 insertions(+), 48 deletions(-) diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 6bcf899c..a6e68c62 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -3,9 +3,9 @@ SET allow_experimental_object_type = 1; -- Create Event CREATE TABLE event ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), --session hostname LowCardinality(String), browser LowCardinality(String), @@ -27,9 +27,9 @@ CREATE TABLE event SETTINGS index_granularity = 8192; CREATE TABLE event_queue ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), url String, referrer String, hostname LowCardinality(String), @@ -52,9 +52,9 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_skip_broken_messages = 1; CREATE MATERIALIZED VIEW event_queue_mv TO event AS -SELECT website_id, - session_uuid, - event_uuid, +SELECT website_id UUID, + session_id UUID, + event_id, url, referrer, hostname, diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 9913c0be..d6b6260b 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -60,6 +60,10 @@ function getDateFormat(date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } +function getCommaSeparatedStringFormat(data, column) { + return data.map(obj => `'${obj[column]}'`).join(','); +} + function getBetweenDates(field, start_at, end_at) { return `${field} between ${getDateFormat(start_at)} and ${getDateFormat(end_at)}`; @@ -180,6 +184,7 @@ export default { getDateStringQuery, getDateQuery, getDateFormat, + getCommaSeparatedStringFormat, getBetweenDates, getFilterQuery, parseFilters, diff --git a/lib/session.js b/lib/session.js index 9e95cb11..61954168 100644 --- a/lib/session.js +++ b/lib/session.js @@ -66,7 +66,7 @@ export async function getSession(req) { if (!sessionId) { try { - session = await createSession(websiteId, { + session = await createSession(websiteId, website_uuid, { session_uuid, hostname, browser, @@ -98,6 +98,7 @@ export async function getSession(req) { return { website_id: websiteId, + website_uuid: website_uuid, session, }; } diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..5899035b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -59,7 +59,7 @@ export default async (req, res) => { await useSession(req, res); const { - session: { website_id, session }, + session: { website_id, website_uuid, session }, } = req; const { type, payload } = getJsonBody(req); @@ -73,9 +73,9 @@ export default async (req, res) => { const event_uuid = uuid(); if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); + await savePageView(website_id, website_uuid, { session, url, referrer }); } else if (type === 'event') { - await saveEvent(website_id, { + await saveEvent(website_id, website_uuid, { session, event_uuid, url, @@ -87,7 +87,12 @@ export default async (req, res) => { } const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + { + website_id, + website_uuid, + session_id: session.session_id, + session_uuid: session.session_uuid, + }, secret(), ); diff --git a/pages/api/websites/[id]/active.js b/pages/api/websites/[id]/active.js index 20550427..845f3b31 100644 --- a/pages/api/websites/[id]/active.js +++ b/pages/api/websites/[id]/active.js @@ -14,8 +14,9 @@ export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const website_uuid = id; - const result = await getActiveVisitors(websiteId); + const result = await getActiveVisitors(websiteId, website_uuid); return ok(res, result); } diff --git a/pages/api/websites/[id]/events.js b/pages/api/websites/[id]/events.js index c633a585..93d01ff9 100644 --- a/pages/api/websites/[id]/events.js +++ b/pages/api/websites/[id]/events.js @@ -21,10 +21,11 @@ export default async (req, res) => { } const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); - const events = await getEventMetrics(websiteId, startDate, endDate, tz, unit, { + const events = await getEventMetrics(websiteId, website_uuid, startDate, endDate, tz, unit, { url, event_name, }); diff --git a/pages/api/websites/[id]/metrics.js b/pages/api/websites/[id]/metrics.js index 206209c6..7b596f8e 100644 --- a/pages/api/websites/[id]/metrics.js +++ b/pages/api/websites/[id]/metrics.js @@ -44,6 +44,7 @@ export default async (req, res) => { const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -106,7 +107,7 @@ export default async (req, res) => { query: type === 'query' && table !== 'event' ? true : undefined, }; - const data = await getPageviewMetrics(websiteId, { + const data = await getPageviewMetrics(websiteId, website_uuid, { startDate, endDate, column, diff --git a/pages/api/websites/[id]/pageviews.js b/pages/api/websites/[id]/pageviews.js index f00fffa1..2b492233 100644 --- a/pages/api/websites/[id]/pageviews.js +++ b/pages/api/websites/[id]/pageviews.js @@ -18,6 +18,7 @@ export default async (req, res) => { req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -26,7 +27,7 @@ export default async (req, res) => { } const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, @@ -41,7 +42,7 @@ export default async (req, res) => { country, }, }), - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, diff --git a/pages/api/websites/[id]/stats.js b/pages/api/websites/[id]/stats.js index a7741af5..54b4ae75 100644 --- a/pages/api/websites/[id]/stats.js +++ b/pages/api/websites/[id]/stats.js @@ -14,6 +14,7 @@ export default async (req, res) => { const { id, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -21,7 +22,7 @@ export default async (req, res) => { const prevStartDate = new Date(+start_at - distance); const prevEndDate = new Date(+end_at - distance); - const metrics = await getWebsiteStats(websiteId, { + const metrics = await getWebsiteStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, filters: { @@ -33,7 +34,7 @@ export default async (req, res) => { country, }, }); - const prevPeriod = await getWebsiteStats(websiteId, { + const prevPeriod = await getWebsiteStats(websiteId, website_uuid, { start_at: prevStartDate, end_at: prevEndDate, filters: { diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index c37f1b39..014b645c 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -36,7 +36,7 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, start_at, end_at, timezone = 'UTC', @@ -44,7 +44,7 @@ async function clickhouseQuery( filters = {}, ) { const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( `select diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 441eed64..7238ecea 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -25,19 +25,23 @@ function relationalQuery(websites, start_at) { } function clickhouseQuery(websites, start_at) { - const { rawQuery, getDateFormat } = clickhouse; + const { rawQuery, getDateFormat, getCommaSeparatedStringFormat } = clickhouse; return rawQuery( `select - event_uuid, + event_id, website_id, - session_uuid, + session_id, created_at, url, event_name from event where event_name != '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 3b10f4cb..a319e646 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -32,14 +32,14 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, event_uuid, url, event_name, event_data }, ) { const { getDateFormat, sendMessage } = kafka; const params = { event_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: event_name?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index bea3502c..0d7b4024 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -34,9 +34,9 @@ async function relationalQuery(website_id, { startDate, endDate, column, table, ); } -async function clickhouseQuery(website_id, { startDate, endDate, column, filters = {} }) { +async function clickhouseQuery(website_uuid, { startDate, endDate, column, filters = {} }) { const { rawQuery, parseFilters, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params); return rawQuery( diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 4f74cf98..c6cc0163 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -45,11 +45,11 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} }, ) { const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( @@ -59,7 +59,7 @@ async function clickhouseQuery( from (select ${getDateQuery('created_at', unit, timezone)} t, - count(${count !== '*' ? 'distinct session_uuid' : count}) y + count(${count !== '*' ? 'distinct session_id' : count}) y from event where event_name = '' and website_id= $1 diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index bc909b72..45a5865f 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -25,15 +25,21 @@ async function relationalQuery(websites, start_at) { } async function clickhouseQuery(websites, start_at) { + const { getCommaSeparatedStringFormat } = clickhouse; + return clickhouse.rawQuery( `select website_id, - session_uuid, + session_id, created_at, url from event where event_name = '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${clickhouse.getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 826d9275..5f861a15 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -22,12 +22,12 @@ async function relationalQuery(website_id, { session: { session_id }, url, refer } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, url, referrer }, ) { const { getDateFormat, sendMessage } = kafka; const params = { - website_id: website_id, + website_id: website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), referrer: referrer?.substring(0, URL_LENGTH), diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 5130b960..0f7d32c6 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -39,14 +39,14 @@ async function relationalQuery(website_id, data) { } async function clickhouseQuery( - website_id, + website_uuid, { session_uuid, hostname, browser, os, screen, language, country, device }, ) { const { getDateFormat, sendMessage } = kafka; const params = { session_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), hostname, browser, diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index efc6a8c2..09cfcdfc 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -23,12 +23,12 @@ async function relationalQuery(website_id) { ); } -async function clickhouseQuery(website_id) { +async function clickhouseQuery(website_uuid) { const { rawQuery, getDateFormat } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( - `select count(distinct session_uuid) x + `select count(distinct session_id) x from event where website_id = $1 and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`, diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 0c3d432a..17ec8d69 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -41,19 +41,19 @@ async function relationalQuery(website_id, { start_at, end_at, filters = {} }) { ); } -async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { +async function clickhouseQuery(website_uuid, { start_at, end_at, filters = {} }) { const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select sum(t.c) as "pageviews", - count(distinct t.session_uuid) as "uniques", + count(distinct t.session_id) as "uniques", sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" from ( - select session_uuid, + select session_id, ${getDateQuery('created_at', 'day')} time_series, count(*) c, min(created_at) min_time, @@ -64,7 +64,7 @@ async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} - group by session_uuid, time_series + group by session_id, time_series ) t;`, params, ); From 78338205a353851ab8c6cfebcd46533a98ca6841 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 10 Oct 2022 13:42:18 -0700 Subject: [PATCH 37/61] Feat/um 62 prisma property names (#1562) * checkpoint * fix pg schema * fix mysql schema * change property names --- components/forms/AccountEditForm.js | 8 +- components/forms/ChangePasswordForm.js | 2 +- components/forms/ShareUrlForm.js | 4 +- components/forms/TrackingCodeForm.js | 2 +- components/forms/WebsiteEditForm.js | 29 ++-- components/layout/Header.js | 2 +- components/metrics/RealtimeChart.js | 4 +- components/metrics/RealtimeHeader.js | 6 +- components/metrics/RealtimeLog.js | 44 +++--- components/metrics/RealtimeViews.js | 6 +- components/metrics/WebsiteChart.js | 2 +- components/metrics/WebsiteHeader.js | 8 +- components/pages/DashboardEdit.js | 8 +- components/pages/RealtimeDashboard.js | 4 +- components/pages/Settings.js | 2 +- components/pages/TestConsole.js | 12 +- components/pages/WebsiteList.js | 8 +- components/settings/AccountSettings.js | 10 +- components/settings/ProfileSettings.js | 4 +- components/settings/WebsiteSettings.js | 34 ++--- db/mysql/schema.prisma | 134 +++++++++--------- db/postgresql/schema.prisma | 134 +++++++++--------- lib/auth.js | 8 +- lib/redis.js | 4 +- lib/session.js | 28 ++-- pages/api/accounts/[id]/index.js | 2 +- pages/api/accounts/[id]/password.js | 2 +- pages/api/accounts/index.js | 6 +- pages/api/auth/login.js | 4 +- pages/api/collect.js | 18 +-- pages/api/realtime/init.js | 6 +- pages/api/share/[id].js | 4 +- pages/api/websites/[id]/events.js | 2 +- pages/api/websites/[id]/index.js | 20 +-- pages/api/websites/[id]/stats.js | 4 +- pages/api/websites/index.js | 27 ++-- pages/console/[[...id]].js | 2 +- queries/admin/account/deleteAccount.js | 22 +-- queries/admin/account/getAccountById.js | 4 +- queries/admin/account/getAccounts.js | 10 +- queries/admin/account/updateAccount.js | 4 +- queries/admin/website/createWebsite.js | 6 +- queries/admin/website/deleteWebsite.js | 18 +-- queries/admin/website/getAllWebsites.js | 2 +- queries/admin/website/getUserWebsites.js | 4 +- queries/admin/website/getWebsiteById.js | 4 +- queries/admin/website/getWebsiteByShareId.js | 4 +- queries/admin/website/getWebsiteByUuid.js | 6 +- queries/admin/website/resetWebsite.js | 12 +- queries/analytics/event/getEventMetrics.js | 8 +- queries/analytics/event/getEvents.js | 4 +- queries/analytics/event/saveEvent.js | 27 ++-- .../analytics/pageview/getPageviewMetrics.js | 8 +- .../analytics/pageview/getPageviewParams.js | 4 +- .../analytics/pageview/getPageviewStats.js | 8 +- queries/analytics/pageview/getPageviews.js | 4 +- queries/analytics/pageview/savePageView.js | 13 +- queries/analytics/session/createSession.js | 20 +-- queries/analytics/session/getSessionByUuid.js | 10 +- .../analytics/session/getSessionMetrics.js | 8 +- queries/analytics/session/getSessions.js | 4 +- queries/analytics/stats/getActiveVisitors.js | 8 +- queries/analytics/stats/getRealtimeData.js | 18 +-- queries/analytics/stats/getWebsiteStats.js | 8 +- tracker/index.js | 12 +- 65 files changed, 431 insertions(+), 433 deletions(-) diff --git a/components/forms/AccountEditForm.js b/components/forms/AccountEditForm.js index bedc3b82..3f54b2f5 100644 --- a/components/forms/AccountEditForm.js +++ b/components/forms/AccountEditForm.js @@ -15,13 +15,13 @@ const initialValues = { password: '', }; -const validate = ({ user_id, username, password }) => { +const validate = ({ userId, username, password }) => { const errors = {}; if (!username) { errors.username = ; } - if (!user_id && !password) { + if (!userId && !password) { errors.password = ; } @@ -33,8 +33,8 @@ export default function AccountEditForm({ values, onSave, onClose }) { const [message, setMessage] = useState(); const handleSubmit = async values => { - const { user_id } = values; - const { ok, data } = await post(user_id ? `/accounts/${user_id}` : '/accounts', values); + const { userId } = values; + const { ok, data } = await post(userId ? `/accounts/${userId}` : '/accounts', values); if (ok) { onSave(); diff --git a/components/forms/ChangePasswordForm.js b/components/forms/ChangePasswordForm.js index 4870d523..4ee657e6 100644 --- a/components/forms/ChangePasswordForm.js +++ b/components/forms/ChangePasswordForm.js @@ -43,7 +43,7 @@ export default function ChangePasswordForm({ values, onSave, onClose }) { const { user } = useUser(); const handleSubmit = async values => { - const { ok, data } = await post(`/accounts/${user.user_id}/password`, values); + const { ok, data } = await post(`/accounts/${user.userId}/password`, values); if (ok) { onSave(); diff --git a/components/forms/ShareUrlForm.js b/components/forms/ShareUrlForm.js index a77d3fbc..9800e54b 100644 --- a/components/forms/ShareUrlForm.js +++ b/components/forms/ShareUrlForm.js @@ -8,7 +8,7 @@ import CopyButton from 'components/common/CopyButton'; export default function TrackingCodeForm({ values, onClose }) { const ref = useRef(); const { basePath } = useRouter(); - const { name, share_id } = values; + const { name, shareId } = values; return ( @@ -27,7 +27,7 @@ export default function TrackingCodeForm({ values, onClose }) { spellCheck={false} defaultValue={`${ document.location.origin - }${basePath}/share/${share_id}/${encodeURIComponent(name)}`} + }${basePath}/share/${shareId}/${encodeURIComponent(name)}`} readOnly /> diff --git a/components/forms/TrackingCodeForm.js b/components/forms/TrackingCodeForm.js index e75260f7..52df3bde 100644 --- a/components/forms/TrackingCodeForm.js +++ b/components/forms/TrackingCodeForm.js @@ -26,7 +26,7 @@ export default function TrackingCodeForm({ values, onClose }) { rows={3} cols={60} spellCheck={false} - defaultValue={``} readOnly diff --git a/components/forms/WebsiteEditForm.js b/components/forms/WebsiteEditForm.js index 337e2d96..00cba540 100644 --- a/components/forms/WebsiteEditForm.js +++ b/components/forms/WebsiteEditForm.js @@ -14,7 +14,6 @@ import useApi from 'hooks/useApi'; import useFetch from 'hooks/useFetch'; import useUser from 'hooks/useUser'; import styles from './WebsiteEditForm.module.css'; -import { getRandomChars } from 'next-basics'; const initialValues = { name: '', @@ -42,14 +41,14 @@ const OwnerDropDown = ({ user, accounts }) => { const { setFieldValue, values } = useFormikContext(); useEffect(() => { - if (values.user_id != null && values.owner === '') { - setFieldValue('owner', values.user_id.toString()); - } else if (user?.user_id && values.owner === '') { - setFieldValue('owner', user.user_id.toString()); + if (values.userId != null && values.owner === '') { + setFieldValue('owner', values.userId.toString()); + } else if (user?.id && values.owner === '') { + setFieldValue('owner', user.id.toString()); } }, [accounts, setFieldValue, user, values]); - if (user?.is_admin) { + if (user?.isAdmin) { return (