From 0387cf0da01ef3abe02ec939835cc22d05087dd6 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 12:49:49 -0700 Subject: [PATCH 01/11] remove CH import --- lib/session.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/session.js b/lib/session.js index 7feafc15..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -4,12 +4,11 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; -import clickhouse from 'lib/clickhouse'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = redis.client; - const hasClickhouse = clickhouse.client; + const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); From 86bd9a7793adb53bc2b0252a53e4339cdc7a4442 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 02/11] remove postgres collect --- lib/session.js | 61 ++++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 47 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -53,49 +52,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, From 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 03/11] break CH fetch --- lib/session.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,7 +2,6 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -36,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 04/11] reject --- lib/session.js | 74 ++++++++++++++++++++++++---------- pages/api/collect.js | 94 +------------------------------------------- 2 files changed, 55 insertions(+), 113 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,12 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -35,16 +37,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index f253d09e..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,95 +1,5 @@ -const { Resolver } = require('dns').promises; -import isbot from 'isbot'; -import ipaddr from 'ipaddr.js'; -import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; -import { savePageView, saveEvent } from 'queries'; -import { useCors, useSession } from 'lib/middleware'; -import { getJsonBody, getIpAddress } from 'lib/request'; -import { secret, uuid } from 'lib/crypto'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - const { type, payload } = getJsonBody(req); - - let { url, referrer, event_name, event_data } = payload; - - if (process.env.REMOVE_TRAILING_SLASH) { - url = url.replace(/\/$/, ''); - } - - const event_uuid = uuid(); - - if (type === 'pageview') { - await savePageView(website_id, { session, url, referrer }); - } else if (type === 'event') { - await saveEvent(website_id, { - session, - event_uuid, - url, - event_name, - event_data, - }); - } else { - return badRequest(res); - } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From 75cba808e3e1edacc2ba105210bf4eff846bab03 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 05/11] do not collect --- pages/api/collect.js | 93 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..d071775b 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,94 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import ipaddr from 'ipaddr.js'; +import isbot from 'isbot'; +import { secret } from 'lib/crypto'; +import { useCors, useSession } from 'lib/middleware'; +import { getIpAddress } from 'lib/request'; +import { createToken, forbidden, send, unauthorized } from 'next-basics'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + // const { type, payload } = getJsonBody(req); + + // let { url, referrer, event_name, event_data } = payload; + + // if (process.env.REMOVE_TRAILING_SLASH) { + // url = url.replace(/\/$/, ''); + // } + + // const event_uuid = uuid(); + + // if (type === 'pageview' && 1 === 0) { + // await savePageView(website_id, { session, url, referrer }); + // } else if (type === 'event' && 1 === 0) { + // await saveEvent(website_id, { + // session, + // event_uuid, + // url, + // event_name, + // event_data, + // }); + // } else { + // return badRequest(res); + // } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 76c0057d2ea885850feec394f9a87ba96ec1e503 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:44:23 -0700 Subject: [PATCH 06/11] Revert "do not collect" This reverts commit 75cba808e3e1edacc2ba105210bf4eff846bab03. --- pages/api/collect.js | 93 +------------------------------------------- 1 file changed, 2 insertions(+), 91 deletions(-) diff --git a/pages/api/collect.js b/pages/api/collect.js index d071775b..66e9a8a4 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,94 +1,5 @@ -const { Resolver } = require('dns').promises; -import ipaddr from 'ipaddr.js'; -import isbot from 'isbot'; -import { secret } from 'lib/crypto'; -import { useCors, useSession } from 'lib/middleware'; -import { getIpAddress } from 'lib/request'; -import { createToken, forbidden, send, unauthorized } from 'next-basics'; +import { forbidden } from 'next-basics'; export default async (req, res) => { - await useCors(req, res); - - if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { - return unauthorized(res); - } - - const ignoreIps = process.env.IGNORE_IP; - const ignoreHostnames = process.env.IGNORE_HOSTNAME; - - if (ignoreIps || ignoreHostnames) { - const ips = []; - - if (ignoreIps) { - ips.push(...ignoreIps.split(',').map(n => n.trim())); - } - - if (ignoreHostnames) { - const resolver = new Resolver(); - const promises = ignoreHostnames - .split(',') - .map(n => resolver.resolve4(n.trim()).catch(() => {})); - - await Promise.all(promises).then(resolvedIps => { - ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); - }); - } - - const clientIp = getIpAddress(req); - - const blocked = ips.find(ip => { - if (ip === clientIp) return true; - - // CIDR notation - if (ip.indexOf('/') > 0) { - const addr = ipaddr.parse(clientIp); - const range = ipaddr.parseCIDR(ip); - - if (addr.kind() === range[0].kind() && addr.match(range)) return true; - } - - return false; - }); - - if (blocked) { - return forbidden(res); - } - } - - await useSession(req, res); - - const { - session: { website_id, session }, - } = req; - - // const { type, payload } = getJsonBody(req); - - // let { url, referrer, event_name, event_data } = payload; - - // if (process.env.REMOVE_TRAILING_SLASH) { - // url = url.replace(/\/$/, ''); - // } - - // const event_uuid = uuid(); - - // if (type === 'pageview' && 1 === 0) { - // await savePageView(website_id, { session, url, referrer }); - // } else if (type === 'event' && 1 === 0) { - // await saveEvent(website_id, { - // session, - // event_uuid, - // url, - // event_name, - // event_data, - // }); - // } else { - // return badRequest(res); - // } - - const token = createToken( - { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, - secret(), - ); - - return send(res, token); + return forbidden(res); }; From f53ea06c231557b66c4809181c71d4d3d01c4820 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:35:38 -0700 Subject: [PATCH 07/11] Revert "reject" This reverts commit 5fd3ba38099212fa87ebadbb6cb8dc3871b362c5. --- lib/session.js | 74 ++++++++++------------------------ pages/api/collect.js | 94 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 55 deletions(-) diff --git a/lib/session.js b/lib/session.js index d635f75a..dd196aaa 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,12 @@ -import { parseToken } from 'next-basics'; -import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -37,14 +35,16 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // Check database if does not exists in Redis - if (!websiteId) { - const website = await getWebsiteByUuid(website_uuid); - websiteId = website ? website.website_id : null; - } + // // Check database if does not exists in Redis + // if (!websiteId) { + // const website = await getWebsiteByUuid(website_uuid); + // websiteId = website ? website.website_id : null; + // } if (!websiteId || websiteId === DELETED) { - throw new Error(`Website not found: ${website_uuid}`); + throw new Error( + `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, + ); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); @@ -53,49 +53,17 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { - // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); - } - - // Check database if does not exists in Redis - if (!sessionId) { - session = await getSessionByUuid(session_uuid); - sessionId = session ? session.session_id : null; - } - - if (!sessionId) { - try { - session = await createSession(websiteId, { - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }); - } catch (e) { - if (!e.message.toLowerCase().includes('unique constraint')) { - throw e; - } - } - } - } else { - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; - } + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; return { website_id: websiteId, diff --git a/pages/api/collect.js b/pages/api/collect.js index 66e9a8a4..f253d09e 100644 --- a/pages/api/collect.js +++ b/pages/api/collect.js @@ -1,5 +1,95 @@ -import { forbidden } from 'next-basics'; +const { Resolver } = require('dns').promises; +import isbot from 'isbot'; +import ipaddr from 'ipaddr.js'; +import { createToken, unauthorized, send, badRequest, forbidden } from 'next-basics'; +import { savePageView, saveEvent } from 'queries'; +import { useCors, useSession } from 'lib/middleware'; +import { getJsonBody, getIpAddress } from 'lib/request'; +import { secret, uuid } from 'lib/crypto'; export default async (req, res) => { - return forbidden(res); + await useCors(req, res); + + if (isbot(req.headers['user-agent']) && !process.env.DISABLE_BOT_CHECK) { + return unauthorized(res); + } + + const ignoreIps = process.env.IGNORE_IP; + const ignoreHostnames = process.env.IGNORE_HOSTNAME; + + if (ignoreIps || ignoreHostnames) { + const ips = []; + + if (ignoreIps) { + ips.push(...ignoreIps.split(',').map(n => n.trim())); + } + + if (ignoreHostnames) { + const resolver = new Resolver(); + const promises = ignoreHostnames + .split(',') + .map(n => resolver.resolve4(n.trim()).catch(() => {})); + + await Promise.all(promises).then(resolvedIps => { + ips.push(...resolvedIps.filter(n => n).flatMap(n => n)); + }); + } + + const clientIp = getIpAddress(req); + + const blocked = ips.find(ip => { + if (ip === clientIp) return true; + + // CIDR notation + if (ip.indexOf('/') > 0) { + const addr = ipaddr.parse(clientIp); + const range = ipaddr.parseCIDR(ip); + + if (addr.kind() === range[0].kind() && addr.match(range)) return true; + } + + return false; + }); + + if (blocked) { + return forbidden(res); + } + } + + await useSession(req, res); + + const { + session: { website_id, session }, + } = req; + + const { type, payload } = getJsonBody(req); + + let { url, referrer, event_name, event_data } = payload; + + if (process.env.REMOVE_TRAILING_SLASH) { + url = url.replace(/\/$/, ''); + } + + const event_uuid = uuid(); + + if (type === 'pageview') { + await savePageView(website_id, { session, url, referrer }); + } else if (type === 'event') { + await saveEvent(website_id, { + session, + event_uuid, + url, + event_name, + event_data, + }); + } else { + return badRequest(res); + } + + const token = createToken( + { website_id, session_id: session.session_id, session_uuid: session.session_uuid }, + secret(), + ); + + return send(res, token); }; From 0e10470c0286d136c5b4a0235f1f36de209d02ba Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:29:55 -0700 Subject: [PATCH 08/11] Revert "break CH fetch" This reverts commit 5dd395918f0658b6bd8ba69dba9cd97751c2c9a9. --- lib/session.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/session.js b/lib/session.js index dd196aaa..f4498596 100644 --- a/lib/session.js +++ b/lib/session.js @@ -2,6 +2,7 @@ import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; import { parseToken } from 'next-basics'; +import { getWebsiteByUuid } from 'queries'; import { validate } from 'uuid'; export async function getSession(req) { @@ -35,16 +36,14 @@ export async function getSession(req) { websiteId = Number(await redis.client.get(`website:${website_uuid}`)); } - // // Check database if does not exists in Redis - // if (!websiteId) { - // const website = await getWebsiteByUuid(website_uuid); - // websiteId = website ? website.website_id : null; - // } + // Check database if does not exists in Redis + if (!websiteId) { + const website = await getWebsiteByUuid(website_uuid); + websiteId = website ? website.website_id : null; + } if (!websiteId || websiteId === DELETED) { - throw new Error( - `Website not found: ${website_uuid} : ${process.env.REDIS_URL} : ${process.env.CLICKHOUSE_URL}`, - ); + throw new Error(`Website not found: ${website_uuid}`); } const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload); From 6bb61d7a53fe11a2ab03161e1b0513792ca3095a Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Thu, 6 Oct 2022 13:13:44 -0700 Subject: [PATCH 09/11] Revert "remove postgres collect" This reverts commit 86bd9a7793adb53bc2b0252a53e4339cdc7a4442. --- lib/session.js | 61 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/session.js b/lib/session.js index f4498596..d635f75a 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,13 +1,14 @@ +import { parseToken } from 'next-basics'; +import { validate } from 'uuid'; import { uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; import { getClientInfo, getJsonBody } from 'lib/request'; -import { parseToken } from 'next-basics'; -import { getWebsiteByUuid } from 'queries'; -import { validate } from 'uuid'; +import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); const hasRedis = process.env.REDIS_URL; + const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -52,17 +53,49 @@ export async function getSession(req) { let sessionId = null; let session = null; - session = { - session_id: sessionId, - session_uuid, - hostname, - browser, - os, - screen, - language, - country, - device, - }; + if (!hasClickhouse) { + // Check if session exists + if (hasRedis) { + sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + } + + // Check database if does not exists in Redis + if (!sessionId) { + session = await getSessionByUuid(session_uuid); + sessionId = session ? session.session_id : null; + } + + if (!sessionId) { + try { + session = await createSession(websiteId, { + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }); + } catch (e) { + if (!e.message.toLowerCase().includes('unique constraint')) { + throw e; + } + } + } + } else { + session = { + session_id: sessionId, + session_uuid, + hostname, + browser, + os, + screen, + language, + country, + device, + }; + } return { website_id: websiteId, From d8c440453cfcda3012e35a5d0f02110f9da3c1bd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 14:18:27 -0700 Subject: [PATCH 10/11] Added getWebsite method. --- lib/auth.js | 8 ++++---- pages/api/websites/[id]/index.js | 6 ++++-- queries/admin/website/getWebsite.js | 7 +++++++ queries/index.js | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 queries/admin/website/getWebsite.js diff --git a/lib/auth.js b/lib/auth.js index b50a923f..b98fb923 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,6 +1,7 @@ +import { validate } from 'uuid'; import { parseSecureToken, parseToken, getItem } from 'next-basics'; import { AUTH_TOKEN, SHARE_TOKEN_HEADER } from './constants'; -import { getWebsiteById } from 'queries'; +import { getWebsite } from 'queries'; import { secret } from './crypto'; export async function getAuthToken(req) { @@ -38,13 +39,12 @@ export async function isValidToken(token, validation) { export async function allowQuery(req, skipToken) { const { id } = req.query; const token = req.headers[SHARE_TOKEN_HEADER]; - const websiteId = +id; - const website = await getWebsiteById(websiteId); + const website = await getWebsite(validate(id) ? { website_uuid: id } : { website_id: +id }); if (website) { if (token && token !== 'undefined' && !skipToken) { - return isValidToken(token, { website_id: websiteId }); + return isValidToken(token, { website_id: website.website_id }); } const authToken = await getAuthToken(req); diff --git a/pages/api/websites/[id]/index.js b/pages/api/websites/[id]/index.js index bc9cb17f..30592213 100644 --- a/pages/api/websites/[id]/index.js +++ b/pages/api/websites/[id]/index.js @@ -1,12 +1,14 @@ import { getRandomChars, methodNotAllowed, ok, unauthorized } from 'next-basics'; -import { deleteWebsite, getWebsiteById, updateWebsite } from 'queries'; +import { deleteWebsite, getWebsite, getWebsiteById, updateWebsite } from 'queries'; import { allowQuery } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; +import { validate } from 'uuid'; export default async (req, res) => { const { id } = req.query; const websiteId = +id; + const where = validate(id) ? { website_uuid: id } : { website_id: +id }; if (req.method === 'GET') { await useCors(req, res); @@ -15,7 +17,7 @@ export default async (req, res) => { return unauthorized(res); } - const website = await getWebsiteById(websiteId); + const website = await getWebsite(where); return ok(res, website); } diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.js new file mode 100644 index 00000000..83c3e83a --- /dev/null +++ b/queries/admin/website/getWebsite.js @@ -0,0 +1,7 @@ +import prisma from 'lib/prisma'; + +export async function getWebsite(where) { + return prisma.client.website.findUnique({ + where, + }); +} diff --git a/queries/index.js b/queries/index.js index 35d79215..d6b4093a 100644 --- a/queries/index.js +++ b/queries/index.js @@ -9,6 +9,7 @@ export * from './admin/website/createWebsite'; export * from './admin/website/deleteWebsite'; export * from './admin/website/getAllWebsites'; export * from './admin/website/getUserWebsites'; +export * from './admin/website/getWebsite'; export * from './admin/website/getWebsiteById'; export * from './admin/website/getWebsiteByShareId'; export * from './admin/website/getWebsiteByUuid'; From e442617421c85c81b6fa9f53e319b65aa2f06fc4 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Oct 2022 15:00:16 -0700 Subject: [PATCH 11/11] Add connect methods to libraries. --- lib/clickhouse.js | 16 +++++++++-- lib/kafka.js | 14 ++++++---- lib/redis.js | 38 +++++++++++++++++++------- lib/session.js | 17 ++++++------ queries/admin/account/deleteAccount.js | 2 +- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/lib/clickhouse.js b/lib/clickhouse.js index 15304214..9913c0be 100644 --- a/lib/clickhouse.js +++ b/lib/clickhouse.js @@ -14,6 +14,9 @@ export const CLICKHOUSE_DATE_FORMATS = { const log = debug('umami:clickhouse'); +let clickhouse; +const enabled = Boolean(process.env.CLICKHOUSE_URL); + function getClient() { const { hostname, @@ -144,6 +147,8 @@ async function rawQuery(query, params = []) { log(formattedQuery); } + await connect(); + return clickhouse.query(formattedQuery).toPromise(); } @@ -159,12 +164,19 @@ async function findFirst(data) { return data[0] ?? null; } -// Initialization -const clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); +async function connect() { + if (!clickhouse) { + clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient()); + } + + return clickhouse; +} export default { + enabled, client: clickhouse, log, + connect, getDateStringQuery, getDateQuery, getDateFormat, diff --git a/lib/kafka.js b/lib/kafka.js index 03833585..782f2803 100644 --- a/lib/kafka.js +++ b/lib/kafka.js @@ -5,6 +5,10 @@ import { KAFKA, KAFKA_PRODUCER } from 'lib/db'; const log = debug('umami:kafka'); +let kafka; +let producer; +const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER); + function getClient() { const { username, password } = new URL(process.env.KAFKA_URL); const brokers = process.env.KAFKA_BROKER.split(','); @@ -61,7 +65,7 @@ function getDateFormat(date) { } async function sendMessage(params, topic) { - await getKafka(); + await connect(); await producer.send({ topic, @@ -74,7 +78,7 @@ async function sendMessage(params, topic) { }); } -async function getKafka() { +async function connect() { if (!kafka) { kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient()); @@ -86,14 +90,12 @@ async function getKafka() { return kafka; } -// Initialization -let kafka; -let producer; - export default { + enabled, client: kafka, producer, log, + connect, getDateFormat, sendMessage, }; diff --git a/lib/redis.js b/lib/redis.js index 63b513e6..15ac27d9 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -8,6 +8,9 @@ const log = debug('umami:redis'); const INITIALIZED = 'redis:initialized'; export const DELETED = 'deleted'; +let redis; +const enabled = Boolean(process.env.REDIS_URL); + function getClient() { if (!process.env.REDIS_URL) { return null; @@ -40,26 +43,41 @@ async function stageData() { return { key: `website:${a.website_uuid}`, value: Number(a.website_id) }; }); - await addRedis(sessionUuids); - await addRedis(websiteIds); + await addSet(sessionUuids); + await addSet(websiteIds); await redis.set(INITIALIZED, 1); } -async function addRedis(ids) { +async function addSet(ids) { for (let i = 0; i < ids.length; i++) { const { key, value } = ids[i]; await redis.set(key, value); } } -// Initialization -const redis = process.env.REDIS_URL && (global[REDIS] || getClient()); +async function get(key) { + await connect(); -(async () => { - if (redis && !(await redis.get(INITIALIZED))) { - await stageData(); + return redis.get(key); +} + +async function set(key, value) { + await connect(); + + return redis.set(key, value); +} + +async function connect() { + if (!redis) { + process.env.REDIS_URL && (global[REDIS] || getClient()); + + if (!(await redis.get(INITIALIZED))) { + await stageData(); + } } -})(); -export default { client: redis, stageData, log }; + return redis; +} + +export default { enabled, client: redis, log, connect, get, set, stageData }; diff --git a/lib/session.js b/lib/session.js index d635f75a..9e95cb11 100644 --- a/lib/session.js +++ b/lib/session.js @@ -1,14 +1,13 @@ import { parseToken } from 'next-basics'; import { validate } from 'uuid'; -import { uuid } from 'lib/crypto'; +import { secret, uuid } from 'lib/crypto'; import redis, { DELETED } from 'lib/redis'; +import clickhouse from 'lib/clickhouse'; import { getClientInfo, getJsonBody } from 'lib/request'; import { createSession, getSessionByUuid, getWebsiteByUuid } from 'queries'; export async function getSession(req) { const { payload } = getJsonBody(req); - const hasRedis = process.env.REDIS_URL; - const hasClickhouse = process.env.CLICKHOUSE_URL; if (!payload) { throw new Error('Invalid request'); @@ -17,7 +16,7 @@ export async function getSession(req) { const cache = req.headers['x-umami-cache']; if (cache) { - const result = await parseToken(cache); + const result = await parseToken(cache, secret()); if (result) { return result; @@ -33,8 +32,8 @@ export async function getSession(req) { let websiteId = null; // Check if website exists - if (hasRedis) { - websiteId = Number(await redis.client.get(`website:${website_uuid}`)); + if (redis.enabled) { + websiteId = Number(await redis.get(`website:${website_uuid}`)); } // Check database if does not exists in Redis @@ -53,10 +52,10 @@ export async function getSession(req) { let sessionId = null; let session = null; - if (!hasClickhouse) { + if (!clickhouse.enabled) { // Check if session exists - if (hasRedis) { - sessionId = Number(await redis.client.get(`session:${session_uuid}`)); + if (redis.enabled) { + sessionId = Number(await redis.get(`session:${session_uuid}`)); } // Check database if does not exists in Redis diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js index 4e204e73..dfe44889 100644 --- a/queries/admin/account/deleteAccount.js +++ b/queries/admin/account/deleteAccount.js @@ -41,7 +41,7 @@ export async function deleteAccount(user_id) { .then(async res => { if (redis.client) { for (let i = 0; i < websiteUuids.length; i++) { - await redis.client.set(`website:${websiteUuids[i]}`, DELETED); + await redis.set(`website:${websiteUuids[i]}`, DELETED); } }