From e8a30e7251fdb6c0819886d21bd75bc109912cf6 Mon Sep 17 00:00:00 2001 From: Sergio Mensing <> Date: Wed, 5 Oct 2022 14:44:49 +0200 Subject: [PATCH 001/147] use slice instead of substring for desired result --- lib/format.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/format.js b/lib/format.js index a85c3ef7..e138831d 100644 --- a/lib/format.js +++ b/lib/format.js @@ -74,7 +74,7 @@ export function stringToColor(str) { let color = '#'; for (let i = 0; i < 3; i++) { let value = (hash >> (i * 8)) & 0xff; - color += ('00' + value.toString(16)).substring(-2); + color += ('00' + value.toString(16)).slice(-2); } return color; } From 0387cf0da01ef3abe02ec939835cc22d05087dd6 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 12:49:49 -0700 Subject: [PATCH 002/147] remove CH import --- lib/session.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/session.js b/lib/session.js index 7feafc15..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -4,12 +4,11 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; -import clickhouse from 'lib/clickhouse'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = redis.client; - const hasClickhouse = clickhouse.client; + const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); From 86bd9a7793adb53bc2b0252a53e4339cdc7a4442 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 003/147] remove postgres collect --- lib/session.js | 61 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -53,49 +52,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, From 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 004/147] break CH fetch --- lib/session.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,7 +2,6 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -36,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 005/147] reject --- lib/session.js | 74 ++++++++++++++++++++++++---------- pages/api/collect.js | 94 +------------------------------------------- 2 files changed, 55 insertions(+), 113 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,12 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -35,16 +37,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,95 +1,5 @@ -const { Resolver } = require('dns').promises; -import isbot from 'isbot'; -import ipaddr from 'ipaddr.js'; -import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; -import { savePageView, saveEvent } from 'queries'; -import { useCors, useSession } from 'lib/middleware'; -import { getJsonBody, getIpAddress } from 'lib/request'; -import { secret, uuid } from 'lib/crypto'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - const { type, payload } = getJsonBody(req); - - let { url, referrer, event_name, event_data } = payload; - - if (process.env.REMOVE_TRAILING_SLASH) { - url = url.replace(/\/$/, ''); - } - - const event_uuid = uuid(); - - if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); - } else if (type === 'event') { - await saveEvent(website_id, { - session, - event_uuid, - url, - event_name, - event_data, - }); - } else { - return badRequest(res); - } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From 75cba808e3e1edacc2ba105210bf4eff846bab03 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 006/147] do not collect --- pages/api/collect.js | 93 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..d071775b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,94 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import ipaddr from 'ipaddr.js'; +import isbot from 'isbot'; +import { secret } from 'lib/crypto'; +import { useCors, useSession } from 'lib/middleware'; +import { getIpAddress } from 'lib/request'; +import { createToken, forbidden, send, unauthorized } from 'next-basics'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + // const { type, payload } = getJsonBody(req); + + // let { url, referrer, event_name, event_data } = payload; + + // if (process.env.REMOVE_TRAILING_SLASH) { + // url = url.replace(/\/$/, ''); + // } + + // const event_uuid = uuid(); + + // if (type === 'pageview' && 1 === 0) { + // await savePageView(website_id, { session, url, referrer }); + // } else if (type === 'event' && 1 === 0) { + // await saveEvent(website_id, { + // session, + // event_uuid, + // url, + // event_name, + // event_data, + // }); + // } else { + // return badRequest(res); + // } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 76c0057d2ea885850feec394f9a87ba96ec1e503 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 007/147] Revert "do not collect" This reverts commit 75cba808e3e1edacc2ba105210bf4eff846bab03. --- pages/api/collect.js | 93 +------------------------------------------- 1 file changed, 2 insertions(+), 91 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index d071775b..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,94 +1,5 @@ -const { Resolver } = require('dns').promises; -import ipaddr from 'ipaddr.js'; -import isbot from 'isbot'; -import { secret } from 'lib/crypto'; -import { useCors, useSession } from 'lib/middleware'; -import { getIpAddress } from 'lib/request'; -import { createToken, forbidden, send, unauthorized } from 'next-basics'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - // const { type, payload } = getJsonBody(req); - - // let { url, referrer, event_name, event_data } = payload; - - // if (process.env.REMOVE_TRAILING_SLASH) { - // url = url.replace(/\/$/, ''); - // } - - // const event_uuid = uuid(); - - // if (type === 'pageview' && 1 === 0) { - // await savePageView(website_id, { session, url, referrer }); - // } else if (type === 'event' && 1 === 0) { - // await saveEvent(website_id, { - // session, - // event_uuid, - // url, - // event_name, - // event_data, - // }); - // } else { - // return badRequest(res); - // } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From f53ea06c231557b66c4809181c71d4d3d01c4820 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 008/147] Revert "reject" This reverts commit 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5. --- lib/session.js | 74 ++++++++++------------------------ pages/api/collect.js | 94 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 55 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,12 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -37,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,49 +53,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..f253d09e 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,95 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import isbot from 'isbot'; +import ipaddr from 'ipaddr.js'; +import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; +import { savePageView, saveEvent } from 'queries'; +import { useCors, useSession } from 'lib/middleware'; +import { getJsonBody, getIpAddress } from 'lib/request'; +import { secret, uuid } from 'lib/crypto'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + const { type, payload } = getJsonBody(req); + + let { url, referrer, event_name, event_data } = payload; + + if (process.env.REMOVE_TRAILING_SLASH) { + url = url.replace(/\/$/, ''); + } + + const event_uuid = uuid(); + + if (type === 'pageview') { + await savePageView(website_id, { session, url, referrer }); + } else if (type === 'event') { + await saveEvent(website_id, { + session, + event_uuid, + url, + event_name, + event_data, + }); + } else { + return badRequest(res); + } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 0e10470c0286d136c5b4a0235f1f36de209d02ba Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 009/147] Revert "break CH fetch" This reverts commit 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9. --- lib/session.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,6 +2,7 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -35,16 +36,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 6bb61d7a53fe11a2ab03161e1b0513792ca3095a Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 010/147] Revert "remove postgres collect" This reverts commit 86bd9a7793adb53bc2b0252a53e4339cdc7a4442. --- lib/session.js | 61 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,13 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -52,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, From d8c440453cfcda3012e35a5d0f02110f9da3c1bd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 14:18:27 -0700 Subject: [PATCH 011/147] Added getWebsite method. --- lib/auth.js | 8 ++++---- pages/api/websites/[id]/index.js | 6 ++++-- queries/admin/website/getWebsite.js | 7 +++++++ queries/index.js | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 queries/admin/website/getWebsite.js diff --git a/lib/auth.js b/lib/auth.js index b50a923f..b98fb923 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,6 +1,7 @@ +import { validate } from 'uuid'; import { parseSecureToken, parseToken, getItem } from 'next-basics'; import { AUTH_TOKEN, SHARE_TOKEN_HEADER } from './constants'; -import { getWebsiteById } from 'queries'; +import { getWebsite } from 'queries'; import { secret } from './crypto'; export async function getAuthToken(req) { @@ -38,13 +39,12 @@ export async function isValidToken(token, validation) { export async function allowQuery(req, skipToken) { const { id } = req.query; const token = req.headers[SHARE_TOKEN_HEADER]; - const websiteId = +id; - const website = await getWebsiteById(websiteId); + const website = await getWebsite(validate(id) ? { website_uuid: id } : { website_id: +id }); if (website) { if (token && token !== 'undefined' && !skipToken) { - return isValidToken(token, { website_id: websiteId }); + return isValidToken(token, { website_id: website.website_id }); } const authToken = await getAuthToken(req); diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index bc9cb17f..30592213 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,12 +1,14 @@ import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; -import { deleteWebsite, getWebsiteById, updateWebsite } from 'queries'; +import { deleteWebsite, getWebsite, getWebsiteById, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; +import { validate } from 'uuid'; export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const where = validate(id) ? { website_uuid: id } : { website_id: +id }; if (req.method === 'GET') { await useCors(req, res); @@ -15,7 +17,7 @@ export default async (req, res) => { return unauthorized(res); } - const website = await getWebsiteById(websiteId); + const website = await getWebsite(where); return ok(res, website); } diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.js new file mode 100644 index 00000000..83c3e83a --- /dev/null +++ b/queries/admin/website/getWebsite.js @@ -0,0 +1,7 @@ +import prisma from 'lib/prisma'; + +export async function getWebsite(where) { + return prisma.client.website.findUnique({ + where, + }); +} diff --git a/queries/index.js b/queries/index.js index 35d79215..d6b4093a 100644 --- a/queries/index.js +++ b/queries/index.js @@ -9,6 +9,7 @@ export * from './admin/website/createWebsite'; export * from './admin/website/deleteWebsite'; export * from './admin/website/getAllWebsites'; export * from './admin/website/getUserWebsites'; +export * from './admin/website/getWebsite'; export * from './admin/website/getWebsiteById'; export * from './admin/website/getWebsiteByShareId'; export * from './admin/website/getWebsiteByUuid'; From e442617421c85c81b6fa9f53e319b65aa2f06fc4 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 15:00:16 -0700 Subject: [PATCH 012/147] Add connect methods to libraries. --- lib/clickhouse.js | 16 +++++++++-- lib/kafka.js | 14 ++++++---- lib/redis.js | 38 +++++++++++++++++++------- lib/session.js | 17 ++++++------ queries/admin/account/deleteAccount.js | 2 +- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 15304214..9913c0be 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -14,6 +14,9 @@ export const CLICKHOUSE_DATE_FORMATS = { const log = debug('umami:clickhouse'); +let clickhouse; +const enabled = Boolean(process.env.CLICKHOUSE_URL); + function getClient() { const { hostname, @@ -144,6 +147,8 @@ async function rawQuery(query, params = []) { log(formattedQuery); } + await connect(); + return clickhouse.query(formattedQuery).toPromise(); } @@ -159,12 +164,19 @@ async function findFirst(data) { return data[0] ?? null; } -// Initialization -const clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); +async function connect() { + if (!clickhouse) { + clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); + } + + return clickhouse; +} export default { + enabled, client: clickhouse, log, + connect, getDateStringQuery, getDateQuery, getDateFormat, diff --git a/lib/kafka.js b/lib/kafka.js index 03833585..782f2803 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -5,6 +5,10 @@ import { KAFKA, KAFKA_PRODUCER } from 'lib/db'; const log = debug('umami:kafka'); +let kafka; +let producer; +const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER); + function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); @@ -61,7 +65,7 @@ function getDateFormat(date) { } async function sendMessage(params, topic) { - await getKafka(); + await connect(); await producer.send({ topic, @@ -74,7 +78,7 @@ async function sendMessage(params, topic) { }); } -async function getKafka() { +async function connect() { if (!kafka) { kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); @@ -86,14 +90,12 @@ async function getKafka() { return kafka; } -// Initialization -let kafka; -let producer; - export default { + enabled, client: kafka, producer, log, + connect, getDateFormat, sendMessage, }; diff --git a/lib/redis.js b/lib/redis.js index 63b513e6..15ac27d9 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -8,6 +8,9 @@ const log = debug('umami:redis'); const INITIALIZED = 'redis:initialized'; export const DELETED = 'deleted'; +let redis; +const enabled = Boolean(process.env.REDIS_URL); + function getClient() { if (!process.env.REDIS_URL) { return null; @@ -40,26 +43,41 @@ async function stageData() { return { key: `website:${a.website_uuid}`, value: Number(a.website_id) }; }); - await addRedis(sessionUuids); - await addRedis(websiteIds); + await addSet(sessionUuids); + await addSet(websiteIds); await redis.set(INITIALIZED, 1); } -async function addRedis(ids) { +async function addSet(ids) { for (let i = 0; i < ids.length; i++) { const { key, value } = ids[i]; await redis.set(key, value); } } -// Initialization -const redis = process.env.REDIS_URL && (global[REDIS] || getClient()); +async function get(key) { + await connect(); -(async () => { - if (redis && !(await redis.get(INITIALIZED))) { - await stageData(); + return redis.get(key); +} + +async function set(key, value) { + await connect(); + + return redis.set(key, value); +} + +async function connect() { + if (!redis) { + process.env.REDIS_URL && (global[REDIS] || getClient()); + + if (!(await redis.get(INITIALIZED))) { + await stageData(); + } } -})(); -export default { client: redis, stageData, log }; + return redis; +} + +export default { enabled, client: redis, log, connect, get, set, stageData }; diff --git a/lib/session.js b/lib/session.js index d635f75a..9e95cb11 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ import { parseToken } from 'next-basics'; import { validate } from 'uuid'; -import { uuid } from 'lib/crypto'; +import { secret, uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; +import clickhouse from 'lib/clickhouse'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -17,7 +16,7 @@ export async function getSession(req) { const cache = req.headers['x-umami-cache']; if (cache) { - const result = await parseToken(cache); + const result = await parseToken(cache, secret()); if (result) { return result; @@ -33,8 +32,8 @@ export async function getSession(req) { let websiteId = null; // Check if website exists - if (hasRedis) { - websiteId = Number(await redis.client.get(`website:${website_uuid}`)); + if (redis.enabled) { + websiteId = Number(await redis.get(`website:${website_uuid}`)); } // Check database if does not exists in Redis @@ -53,10 +52,10 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { + if (!clickhouse.enabled) { // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + if (redis.enabled) { + sessionId = Number(await redis.get(`session:${session_uuid}`)); } // Check database if does not exists in Redis diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js index 4e204e73..dfe44889 100644 --- a/queries/admin/account/deleteAccount.js +++ b/queries/admin/account/deleteAccount.js @@ -41,7 +41,7 @@ export async function deleteAccount(user_id) { .then(async res => { if (redis.client) { for (let i = 0; i < websiteUuids.length; i++) { - await redis.client.set(`website:${websiteUuids[i]}`, DELETED); + await redis.set(`website:${websiteUuids[i]}`, DELETED); } } From dcf16e1411e80bba2abfc25695d1c0585fa8d45e Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 16:41:37 -0700 Subject: [PATCH 013/147] fix redis init --- lib/redis.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/redis.js b/lib/redis.js index 15ac27d9..01a9e3c6 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -70,11 +70,7 @@ async function set(key, value) { async function connect() { if (!redis) { - process.env.REDIS_URL && (global[REDIS] || getClient()); - - if (!(await redis.get(INITIALIZED))) { - await stageData(); - } + redis = process.env.REDIS_URL && (global[REDIS] || getClient()); } return redis; From 2428314f584e5e21bf9764400bab6b3608f1d4ed Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 7 Oct 2022 17:13:03 -0700 Subject: [PATCH 014/147] Updated save method for websites. --- components/forms/WebsiteEditForm.js | 7 ++++-- pages/api/websites/[id]/index.js | 33 +++++++++++++++----------- queries/admin/website/updateWebsite.js | 6 ++--- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/components/forms/WebsiteEditForm.js b/components/forms/WebsiteEditForm.js index 21daeeaf..337e2d96 100644 --- a/components/forms/WebsiteEditForm.js +++ b/components/forms/WebsiteEditForm.js @@ -14,6 +14,7 @@ import useApi from 'hooks/useApi'; import useFetch from 'hooks/useFetch'; import useUser from 'hooks/useUser'; import styles from './WebsiteEditForm.module.css'; +import { getRandomChars } from 'next-basics'; const initialValues = { name: '', @@ -78,7 +79,10 @@ export default function WebsiteEditForm({ values, onSave, onClose }) { const [message, setMessage] = useState(); const handleSubmit = async values => { - const { website_id } = values; + const { website_id, enable_share_url, share_id } = values; + if (enable_share_url) { + values.share_id = share_id || getRandomChars(8); + } const { ok, data } = await post(website_id ? `/websites/${website_id}` : '/websites', values); if (ok) { @@ -137,7 +141,6 @@ export default function WebsiteEditForm({ values, onSave, onClose }) { defaultMessage="Enable share URL" /> } - value={null} /> )} diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index 30592213..8234a5f2 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,5 +1,5 @@ -import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; -import { deleteWebsite, getWebsite, getWebsiteById, updateWebsite } from 'queries'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { deleteWebsite, getAccount, getWebsite, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; import { validate } from 'uuid'; @@ -25,24 +25,29 @@ export default async (req, res) => { if (req.method === 'POST') { await useAuth(req, res); - const { is_admin: currentUserIsAdmin, user_id: currentUserId } = req.auth; - const { name, domain, owner, enable_share_url } = req.body; + const { is_admin: currentUserIsAdmin, user_id: currentUserId, account_uuid } = req.auth; + const { name, domain, owner, share_id } = req.body; + let account; - const website = await getWebsiteById(websiteId); + if (account_uuid) { + account = await getAccount({ account_uuid }); + } + + const website = await getWebsite(where); if (website.user_id !== currentUserId && !currentUserIsAdmin) { return unauthorized(res); } - let { share_id } = website; - - if (enable_share_url) { - share_id = share_id ? share_id : getRandomChars(8); - } else { - share_id = null; - } - - await updateWebsite(websiteId, { name, domain, share_id, user_id: +owner }); + await updateWebsite( + { + name, + domain, + share_id: share_id || null, + user_id: account ? account.id : +owner, + }, + where, + ); return ok(res); } diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.js index 54f01f66..1a5079a9 100644 --- a/queries/admin/website/updateWebsite.js +++ b/queries/admin/website/updateWebsite.js @@ -1,10 +1,8 @@ import prisma from 'lib/prisma'; -export async function updateWebsite(website_id, data) { +export async function updateWebsite(data, where) { return prisma.client.website.update({ - where: { - website_id, - }, + where, data, }); } From 6f6cbaa09805af605520c9a0b499aab2f1c0a57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=B2=97?= Date: Sat, 8 Oct 2022 13:46:50 +0800 Subject: [PATCH 015/147] Update getWebsiteByUuid.js if redis website set from getWebsiteByUuid,the value is 1 forever --- queries/admin/website/getWebsiteByUuid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.js index 900e4539..1b1e04ca 100644 --- a/queries/admin/website/getWebsiteByUuid.js +++ b/queries/admin/website/getWebsiteByUuid.js @@ -10,7 +10,7 @@ export async function getWebsiteByUuid(website_uuid) { }) .then(async res => { if (redis.client && res) { - await redis.client.set(`website:${res.website_uuid}`, 1); + await redis.client.set(`website:${res.website_uuid}`, res.website_id); } return res; From 96add409b69f76a1a910199143b87957017107f1 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Sat, 8 Oct 2022 16:12:33 -0700 Subject: [PATCH 016/147] update api to new CH columns --- db/clickhouse/schema.sql | 18 +++++++++--------- lib/clickhouse.js | 5 +++++ lib/session.js | 3 ++- pages/api/collect.js | 13 +++++++++---- pages/api/websites/[id]/active.js | 3 ++- pages/api/websites/[id]/events.js | 3 ++- pages/api/websites/[id]/metrics.js | 3 ++- pages/api/websites/[id]/pageviews.js | 5 +++-- pages/api/websites/[id]/stats.js | 5 +++-- queries/analytics/event/getEventMetrics.js | 4 ++-- queries/analytics/event/getEvents.js | 12 ++++++++---- queries/analytics/event/saveEvent.js | 4 ++-- .../analytics/pageview/getPageviewMetrics.js | 4 ++-- queries/analytics/pageview/getPageviewStats.js | 6 +++--- queries/analytics/pageview/getPageviews.js | 10 ++++++++-- queries/analytics/pageview/savePageView.js | 4 ++-- queries/analytics/session/createSession.js | 4 ++-- queries/analytics/stats/getActiveVisitors.js | 6 +++--- queries/analytics/stats/getWebsiteStats.js | 10 +++++----- 19 files changed, 74 insertions(+), 48 deletions(-) diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 6bcf899c..a6e68c62 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -3,9 +3,9 @@ SET allow_experimental_object_type = 1; -- Create Event CREATE TABLE event ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), --session hostname LowCardinality(String), browser LowCardinality(String), @@ -27,9 +27,9 @@ CREATE TABLE event SETTINGS index_granularity = 8192; CREATE TABLE event_queue ( - website_id UInt32, - session_uuid UUID, - event_uuid Nullable(UUID), + website_id UUID, + session_id UUID, + event_id Nullable(UUID), url String, referrer String, hostname LowCardinality(String), @@ -52,9 +52,9 @@ SETTINGS kafka_broker_list = 'domain:9092,domain:9093,domain:9094', -- input bro kafka_skip_broken_messages = 1; CREATE MATERIALIZED VIEW event_queue_mv TO event AS -SELECT website_id, - session_uuid, - event_uuid, +SELECT website_id UUID, + session_id UUID, + event_id, url, referrer, hostname, diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 9913c0be..d6b6260b 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -60,6 +60,10 @@ function getDateFormat(date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } +function getCommaSeparatedStringFormat(data, column) { + return data.map(obj => `'${obj[column]}'`).join(','); +} + function getBetweenDates(field, start_at, end_at) { return `${field} between ${getDateFormat(start_at)} and ${getDateFormat(end_at)}`; @@ -180,6 +184,7 @@ export default { getDateStringQuery, getDateQuery, getDateFormat, + getCommaSeparatedStringFormat, getBetweenDates, getFilterQuery, parseFilters, diff --git a/lib/session.js b/lib/session.js index 9e95cb11..61954168 100644 --- a/lib/session.js +++ b/lib/session.js @@ -66,7 +66,7 @@ export async function getSession(req) { if (!sessionId) { try { - session = await createSession(websiteId, { + session = await createSession(websiteId, website_uuid, { session_uuid, hostname, browser, @@ -98,6 +98,7 @@ export async function getSession(req) { return { website_id: websiteId, + website_uuid: website_uuid, session, }; } diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..5899035b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -59,7 +59,7 @@ export default async (req, res) => { await useSession(req, res); const { - session: { website_id, session }, + session: { website_id, website_uuid, session }, } = req; const { type, payload } = getJsonBody(req); @@ -73,9 +73,9 @@ export default async (req, res) => { const event_uuid = uuid(); if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); + await savePageView(website_id, website_uuid, { session, url, referrer }); } else if (type === 'event') { - await saveEvent(website_id, { + await saveEvent(website_id, website_uuid, { session, event_uuid, url, @@ -87,7 +87,12 @@ export default async (req, res) => { } const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + { + website_id, + website_uuid, + session_id: session.session_id, + session_uuid: session.session_uuid, + }, secret(), ); diff --git a/pages/api/websites/[id]/active.js b/pages/api/websites/[id]/active.js index 20550427..845f3b31 100644 --- a/pages/api/websites/[id]/active.js +++ b/pages/api/websites/[id]/active.js @@ -14,8 +14,9 @@ export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const website_uuid = id; - const result = await getActiveVisitors(websiteId); + const result = await getActiveVisitors(websiteId, website_uuid); return ok(res, result); } diff --git a/pages/api/websites/[id]/events.js b/pages/api/websites/[id]/events.js index c633a585..93d01ff9 100644 --- a/pages/api/websites/[id]/events.js +++ b/pages/api/websites/[id]/events.js @@ -21,10 +21,11 @@ export default async (req, res) => { } const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); - const events = await getEventMetrics(websiteId, startDate, endDate, tz, unit, { + const events = await getEventMetrics(websiteId, website_uuid, startDate, endDate, tz, unit, { url, event_name, }); diff --git a/pages/api/websites/[id]/metrics.js b/pages/api/websites/[id]/metrics.js index 206209c6..7b596f8e 100644 --- a/pages/api/websites/[id]/metrics.js +++ b/pages/api/websites/[id]/metrics.js @@ -44,6 +44,7 @@ export default async (req, res) => { const { id, type, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -106,7 +107,7 @@ export default async (req, res) => { query: type === 'query' && table !== 'event' ? true : undefined, }; - const data = await getPageviewMetrics(websiteId, { + const data = await getPageviewMetrics(websiteId, website_uuid, { startDate, endDate, column, diff --git a/pages/api/websites/[id]/pageviews.js b/pages/api/websites/[id]/pageviews.js index f00fffa1..2b492233 100644 --- a/pages/api/websites/[id]/pageviews.js +++ b/pages/api/websites/[id]/pageviews.js @@ -18,6 +18,7 @@ export default async (req, res) => { req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -26,7 +27,7 @@ export default async (req, res) => { } const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, @@ -41,7 +42,7 @@ export default async (req, res) => { country, }, }), - getPageviewStats(websiteId, { + getPageviewStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, timezone: tz, diff --git a/pages/api/websites/[id]/stats.js b/pages/api/websites/[id]/stats.js index a7741af5..54b4ae75 100644 --- a/pages/api/websites/[id]/stats.js +++ b/pages/api/websites/[id]/stats.js @@ -14,6 +14,7 @@ export default async (req, res) => { const { id, start_at, end_at, url, referrer, os, browser, device, country } = req.query; const websiteId = +id; + const website_uuid = id; const startDate = new Date(+start_at); const endDate = new Date(+end_at); @@ -21,7 +22,7 @@ export default async (req, res) => { const prevStartDate = new Date(+start_at - distance); const prevEndDate = new Date(+end_at - distance); - const metrics = await getWebsiteStats(websiteId, { + const metrics = await getWebsiteStats(websiteId, website_uuid, { start_at: startDate, end_at: endDate, filters: { @@ -33,7 +34,7 @@ export default async (req, res) => { country, }, }); - const prevPeriod = await getWebsiteStats(websiteId, { + const prevPeriod = await getWebsiteStats(websiteId, website_uuid, { start_at: prevStartDate, end_at: prevEndDate, filters: { diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index c37f1b39..014b645c 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -36,7 +36,7 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, start_at, end_at, timezone = 'UTC', @@ -44,7 +44,7 @@ async function clickhouseQuery( filters = {}, ) { const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( `select diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js index 441eed64..7238ecea 100644 --- a/queries/analytics/event/getEvents.js +++ b/queries/analytics/event/getEvents.js @@ -25,19 +25,23 @@ function relationalQuery(websites, start_at) { } function clickhouseQuery(websites, start_at) { - const { rawQuery, getDateFormat } = clickhouse; + const { rawQuery, getDateFormat, getCommaSeparatedStringFormat } = clickhouse; return rawQuery( `select - event_uuid, + event_id, website_id, - session_uuid, + session_id, created_at, url, event_name from event where event_name != '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index 3b10f4cb..a319e646 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -32,14 +32,14 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_ } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, event_uuid, url, event_name, event_data }, ) { const { getDateFormat, sendMessage } = kafka; const params = { event_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: event_name?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index bea3502c..0d7b4024 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -34,9 +34,9 @@ async function relationalQuery(website_id, { startDate, endDate, column, table, ); } -async function clickhouseQuery(website_id, { startDate, endDate, column, filters = {} }) { +async function clickhouseQuery(website_uuid, { startDate, endDate, column, filters = {} }) { const { rawQuery, parseFilters, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params); return rawQuery( diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 4f74cf98..c6cc0163 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -45,11 +45,11 @@ async function relationalQuery( } async function clickhouseQuery( - website_id, + website_uuid, { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} }, ) { const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( @@ -59,7 +59,7 @@ async function clickhouseQuery( from (select ${getDateQuery('created_at', unit, timezone)} t, - count(${count !== '*' ? 'distinct session_uuid' : count}) y + count(${count !== '*' ? 'distinct session_id' : count}) y from event where event_name = '' and website_id= $1 diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js index bc909b72..45a5865f 100644 --- a/queries/analytics/pageview/getPageviews.js +++ b/queries/analytics/pageview/getPageviews.js @@ -25,15 +25,21 @@ async function relationalQuery(websites, start_at) { } async function clickhouseQuery(websites, start_at) { + const { getCommaSeparatedStringFormat } = clickhouse; + return clickhouse.rawQuery( `select website_id, - session_uuid, + session_id, created_at, url from event where event_name = '' - and ${websites && websites.length > 0 ? `website_id in (${websites.join(',')})` : '0 = 0'} + and ${ + websites && websites.length > 0 + ? `website_id in (${getCommaSeparatedStringFormat(websites, websites.website_uuid)})` + : '0 = 0' + } and created_at >= ${clickhouse.getDateFormat(start_at)}`, ); } diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index 826d9275..5f861a15 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -22,12 +22,12 @@ async function relationalQuery(website_id, { session: { session_id }, url, refer } async function clickhouseQuery( - website_id, + website_uuid, { session: { country, ...sessionArgs }, url, referrer }, ) { const { getDateFormat, sendMessage } = kafka; const params = { - website_id: website_id, + website_id: website_uuid, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), referrer: referrer?.substring(0, URL_LENGTH), diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index 5130b960..0f7d32c6 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -39,14 +39,14 @@ async function relationalQuery(website_id, data) { } async function clickhouseQuery( - website_id, + website_uuid, { session_uuid, hostname, browser, os, screen, language, country, device }, ) { const { getDateFormat, sendMessage } = kafka; const params = { session_uuid, - website_id, + website_uuid, created_at: getDateFormat(new Date()), hostname, browser, diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js index efc6a8c2..09cfcdfc 100644 --- a/queries/analytics/stats/getActiveVisitors.js +++ b/queries/analytics/stats/getActiveVisitors.js @@ -23,12 +23,12 @@ async function relationalQuery(website_id) { ); } -async function clickhouseQuery(website_id) { +async function clickhouseQuery(website_uuid) { const { rawQuery, getDateFormat } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; return rawQuery( - `select count(distinct session_uuid) x + `select count(distinct session_id) x from event where website_id = $1 and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`, diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js index 0c3d432a..17ec8d69 100644 --- a/queries/analytics/stats/getWebsiteStats.js +++ b/queries/analytics/stats/getWebsiteStats.js @@ -41,19 +41,19 @@ async function relationalQuery(website_id, { start_at, end_at, filters = {} }) { ); } -async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { +async function clickhouseQuery(website_uuid, { start_at, end_at, filters = {} }) { const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse; - const params = [website_id]; + const params = [website_uuid]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select sum(t.c) as "pageviews", - count(distinct t.session_uuid) as "uniques", + count(distinct t.session_id) as "uniques", sum(if(t.c = 1, 1, 0)) as "bounces", sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime" from ( - select session_uuid, + select session_id, ${getDateQuery('created_at', 'day')} time_series, count(*) c, min(created_at) min_time, @@ -64,7 +64,7 @@ async function clickhouseQuery(website_id, { start_at, end_at, filters = {} }) { and ${getBetweenDates('created_at', start_at, end_at)} ${pageviewQuery} ${sessionQuery} - group by session_uuid, time_series + group by session_id, time_series ) t;`, params, ); From 78338205a353851ab8c6cfebcd46533a98ca6841 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Mon, 10 Oct 2022 13:42:18 -0700 Subject: [PATCH 017/147] Feat/um 62 prisma property names (#1562) * checkpoint * fix pg schema * fix mysql schema * change property names --- components/forms/AccountEditForm.js | 8 +- components/forms/ChangePasswordForm.js | 2 +- components/forms/ShareUrlForm.js | 4 +- components/forms/TrackingCodeForm.js | 2 +- components/forms/WebsiteEditForm.js | 29 ++-- components/layout/Header.js | 2 +- components/metrics/RealtimeChart.js | 4 +- components/metrics/RealtimeHeader.js | 6 +- components/metrics/RealtimeLog.js | 44 +++--- components/metrics/RealtimeViews.js | 6 +- components/metrics/WebsiteChart.js | 2 +- components/metrics/WebsiteHeader.js | 8 +- components/pages/DashboardEdit.js | 8 +- components/pages/RealtimeDashboard.js | 4 +- components/pages/Settings.js | 2 +- components/pages/TestConsole.js | 12 +- components/pages/WebsiteList.js | 8 +- components/settings/AccountSettings.js | 10 +- components/settings/ProfileSettings.js | 4 +- components/settings/WebsiteSettings.js | 34 ++--- db/mysql/schema.prisma | 134 +++++++++--------- db/postgresql/schema.prisma | 134 +++++++++--------- lib/auth.js | 8 +- lib/redis.js | 4 +- lib/session.js | 28 ++-- pages/api/accounts/[id]/index.js | 2 +- pages/api/accounts/[id]/password.js | 2 +- pages/api/accounts/index.js | 6 +- pages/api/auth/login.js | 4 +- pages/api/collect.js | 18 +-- pages/api/realtime/init.js | 6 +- pages/api/share/[id].js | 4 +- pages/api/websites/[id]/events.js | 2 +- pages/api/websites/[id]/index.js | 20 +-- pages/api/websites/[id]/stats.js | 4 +- pages/api/websites/index.js | 27 ++-- pages/console/[[...id]].js | 2 +- queries/admin/account/deleteAccount.js | 22 +-- queries/admin/account/getAccountById.js | 4 +- queries/admin/account/getAccounts.js | 10 +- queries/admin/account/updateAccount.js | 4 +- queries/admin/website/createWebsite.js | 6 +- queries/admin/website/deleteWebsite.js | 18 +-- queries/admin/website/getAllWebsites.js | 2 +- queries/admin/website/getUserWebsites.js | 4 +- queries/admin/website/getWebsiteById.js | 4 +- queries/admin/website/getWebsiteByShareId.js | 4 +- queries/admin/website/getWebsiteByUuid.js | 6 +- queries/admin/website/resetWebsite.js | 12 +- queries/analytics/event/getEventMetrics.js | 8 +- queries/analytics/event/getEvents.js | 4 +- queries/analytics/event/saveEvent.js | 27 ++-- .../analytics/pageview/getPageviewMetrics.js | 8 +- .../analytics/pageview/getPageviewParams.js | 4 +- .../analytics/pageview/getPageviewStats.js | 8 +- queries/analytics/pageview/getPageviews.js | 4 +- queries/analytics/pageview/savePageView.js | 13 +- queries/analytics/session/createSession.js | 20 +-- queries/analytics/session/getSessionByUuid.js | 10 +- .../analytics/session/getSessionMetrics.js | 8 +- queries/analytics/session/getSessions.js | 4 +- queries/analytics/stats/getActiveVisitors.js | 8 +- queries/analytics/stats/getRealtimeData.js | 18 +-- queries/analytics/stats/getWebsiteStats.js | 8 +- tracker/index.js | 12 +- 65 files changed, 431 insertions(+), 433 deletions(-) diff --git a/components/forms/AccountEditForm.js b/components/forms/AccountEditForm.js index bedc3b82..3f54b2f5 100644 --- a/components/forms/AccountEditForm.js +++ b/components/forms/AccountEditForm.js @@ -15,13 +15,13 @@ const initialValues = { password: '', }; -const validate = ({ user_id, username, password }) => { +const validate = ({ userId, username, password }) => { const errors = {}; if (!username) { errors.username = ; } - if (!user_id && !password) { + if (!userId && !password) { errors.password = ; } @@ -33,8 +33,8 @@ export default function AccountEditForm({ values, onSave, onClose }) { const [message, setMessage] = useState(); const handleSubmit = async values => { - const { user_id } = values; - const { ok, data } = await post(user_id ? `/accounts/${user_id}` : '/accounts', values); + const { userId } = values; + const { ok, data } = await post(userId ? `/accounts/${userId}` : '/accounts', values); if (ok) { onSave(); diff --git a/components/forms/ChangePasswordForm.js b/components/forms/ChangePasswordForm.js index 4870d523..4ee657e6 100644 --- a/components/forms/ChangePasswordForm.js +++ b/components/forms/ChangePasswordForm.js @@ -43,7 +43,7 @@ export default function ChangePasswordForm({ values, onSave, onClose }) { const { user } = useUser(); const handleSubmit = async values => { - const { ok, data } = await post(`/accounts/${user.user_id}/password`, values); + const { ok, data } = await post(`/accounts/${user.userId}/password`, values); if (ok) { onSave(); diff --git a/components/forms/ShareUrlForm.js b/components/forms/ShareUrlForm.js index a77d3fbc..9800e54b 100644 --- a/components/forms/ShareUrlForm.js +++ b/components/forms/ShareUrlForm.js @@ -8,7 +8,7 @@ import CopyButton from 'components/common/CopyButton'; export default function TrackingCodeForm({ values, onClose }) { const ref = useRef(); const { basePath } = useRouter(); - const { name, share_id } = values; + const { name, shareId } = values; return ( @@ -27,7 +27,7 @@ export default function TrackingCodeForm({ values, onClose }) { spellCheck={false} defaultValue={`${ document.location.origin - }${basePath}/share/${share_id}/${encodeURIComponent(name)}`} + }${basePath}/share/${shareId}/${encodeURIComponent(name)}`} readOnly /> diff --git a/components/forms/TrackingCodeForm.js b/components/forms/TrackingCodeForm.js index e75260f7..52df3bde 100644 --- a/components/forms/TrackingCodeForm.js +++ b/components/forms/TrackingCodeForm.js @@ -26,7 +26,7 @@ export default function TrackingCodeForm({ values, onClose }) { rows={3} cols={60} spellCheck={false} - defaultValue={``} readOnly diff --git a/components/forms/WebsiteEditForm.js b/components/forms/WebsiteEditForm.js index 337e2d96..00cba540 100644 --- a/components/forms/WebsiteEditForm.js +++ b/components/forms/WebsiteEditForm.js @@ -14,7 +14,6 @@ import useApi from 'hooks/useApi'; import useFetch from 'hooks/useFetch'; import useUser from 'hooks/useUser'; import styles from './WebsiteEditForm.module.css'; -import { getRandomChars } from 'next-basics'; const initialValues = { name: '', @@ -42,14 +41,14 @@ const OwnerDropDown = ({ user, accounts }) => { const { setFieldValue, values } = useFormikContext(); useEffect(() => { - if (values.user_id != null && values.owner === '') { - setFieldValue('owner', values.user_id.toString()); - } else if (user?.user_id && values.owner === '') { - setFieldValue('owner', user.user_id.toString()); + if (values.userId != null && values.owner === '') { + setFieldValue('owner', values.userId.toString()); + } else if (user?.id && values.owner === '') { + setFieldValue('owner', user.id.toString()); } }, [accounts, setFieldValue, user, values]); - if (user?.is_admin) { + if (user?.isAdmin) { return (