From 3485b6268bc5976e439dbf2795b63e04119035d9 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Mon, 7 Nov 2022 16:22:49 -0800 Subject: [PATCH] Added rev_id column. Updated redis calls. --- db/postgresql/schema.prisma | 2 +- lib/auth.js | 15 ++++- lib/cache.js | 0 lib/redis.js | 8 ++- pages/api/auth/login.js | 9 +++ pages/api/auth/logout.js | 18 ++++++ pages/logout.js | 10 +++- queries/admin/website/createWebsite.js | 2 +- queries/admin/website/getWebsite.js | 2 +- queries/analytics/event/getEventData.js | 25 ++++----- queries/analytics/event/getEventMetrics.js | 7 ++- queries/analytics/event/saveEvent.js | 3 + .../analytics/pageview/getPageviewMetrics.js | 7 ++- .../analytics/pageview/getPageviewStats.js | 4 +- queries/analytics/pageview/savePageView.js | 3 + queries/analytics/session/createSession.js | 56 +++++++++---------- queries/analytics/session/getSession.js | 34 ++++------- .../analytics/session/getSessionMetrics.js | 7 ++- 18 files changed, 133 insertions(+), 79 deletions(-) create mode 100644 lib/cache.js create mode 100644 pages/api/auth/logout.js diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 9e71377a..93b18608 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -45,7 +45,7 @@ model website { name String @db.VarChar(100) domain String? @db.VarChar(500) shareId String? @unique @map("share_id") @db.VarChar(64) - revId Int @default(0) @map("rev_id") @db.Int + revId Int @default(0) @map("rev_id") @db.Integer createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) isDeleted Boolean @default(false) @map("is_deleted") diff --git a/lib/auth.js b/lib/auth.js index f843065c..ae66ce81 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -1,4 +1,4 @@ -import { parseSecureToken, parseToken } from 'next-basics'; +import { getRandomChars, parseSecureToken, parseToken } from 'next-basics'; import { getUser, getWebsite } from 'queries'; import debug from 'debug'; import { SHARE_TOKEN_HEADER, TYPE_USER, TYPE_WEBSITE } from 'lib/constants'; @@ -6,10 +6,19 @@ import { secret } from 'lib/crypto'; const log = debug('umami:auth'); +export function generateAuthToken() { + return getRandomChars(32); +} + +export function getAuthToken(req) { + const token = req.headers.authorization; + + return token.split(' ')[1]; +} + export function parseAuthToken(req) { try { - const token = req.headers.authorization; - return parseSecureToken(token.split(' ')[1], secret()); + return parseSecureToken(getAuthToken(req), secret()); } catch (e) { log(e); return null; diff --git a/lib/cache.js b/lib/cache.js new file mode 100644 index 00000000..e69de29b diff --git a/lib/redis.js b/lib/redis.js index 16cfce19..6c4e58ef 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -41,6 +41,12 @@ async function set(key, value) { return redis.set(key, value); } +async function del(key) { + await connect(); + + return redis.del(key); +} + async function connect() { if (!redis) { redis = process.env.REDIS_URL && (global[REDIS] || getClient()); @@ -49,4 +55,4 @@ async function connect() { return redis; } -export default { enabled, client: redis, log, connect, get, set }; +export default { enabled, client: redis, log, connect, get, set, del }; diff --git a/pages/api/auth/login.js b/pages/api/auth/login.js index fb9b5a1b..dfdd3ec3 100644 --- a/pages/api/auth/login.js +++ b/pages/api/auth/login.js @@ -1,6 +1,8 @@ import { ok, unauthorized, badRequest, checkPassword, createSecureToken } from 'next-basics'; import { getUser } from 'queries'; import { secret } from 'lib/crypto'; +import redis from 'lib/redis'; +import { generateAuthToken } from 'lib/auth'; export default async (req, res) => { const { username, password } = req.body; @@ -13,6 +15,13 @@ export default async (req, res) => { if (user && checkPassword(password, user.password)) { const { id: userId, username, isAdmin } = user; + + if (redis.enabled) { + const token = `auth:${generateAuthToken()}`; + + return ok(res, { token, user }); + } + const token = createSecureToken({ userId, username, isAdmin }, secret()); return ok(res, { token, user }); diff --git a/pages/api/auth/logout.js b/pages/api/auth/logout.js new file mode 100644 index 00000000..88ea81ec --- /dev/null +++ b/pages/api/auth/logout.js @@ -0,0 +1,18 @@ +import { methodNotAllowed, ok } from 'next-basics'; +import { useAuth } from 'lib/middleware'; +import redis from 'lib/redis'; +import { getAuthToken } from 'lib/auth'; + +export default async (req, res) => { + await useAuth(req, res); + + if (req.method === 'POST') { + if (redis.enabled) { + await redis.del(`auth:${getAuthToken(req)}`); + } + + return ok(res); + } + + return methodNotAllowed(res); +}; diff --git a/pages/logout.js b/pages/logout.js index bcc99a10..a91205a4 100644 --- a/pages/logout.js +++ b/pages/logout.js @@ -1,14 +1,22 @@ import { useEffect } from 'react'; import { useRouter } from 'next/router'; -import { removeItem } from 'next-basics'; +import { removeItem, useApi } from 'next-basics'; import { AUTH_TOKEN } from 'lib/constants'; import { setUser } from 'store/app'; export default function LogoutPage() { const router = useRouter(); + const { post } = useApi(); useEffect(() => { + async function logout() { + await post('/logout'); + } + removeItem(AUTH_TOKEN); + + logout(); + router.push('/login'); return () => setUser(null); diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js index f9cb94be..02bb2838 100644 --- a/queries/admin/website/createWebsite.js +++ b/queries/admin/website/createWebsite.js @@ -15,7 +15,7 @@ export async function createWebsite(userId, data) { }) .then(async res => { if (redis.enabled && res) { - await redis.set(`website:${res.id}`, 1); + await redis.set(`website:${res.id}`, res); } return res; diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.js index b7a468d2..46ec5114 100644 --- a/queries/admin/website/getWebsite.js +++ b/queries/admin/website/getWebsite.js @@ -8,7 +8,7 @@ export async function getWebsite(where) { }) .then(async data => { if (redis.enabled && data) { - await redis.set(`website:${data.id}`, 1); + await redis.set(`website:${data.id}`, data); } return data; diff --git a/queries/analytics/event/getEventData.js b/queries/analytics/event/getEventData.js index 72857af4..df2f7269 100644 --- a/queries/analytics/event/getEventData.js +++ b/queries/analytics/event/getEventData.js @@ -1,11 +1,16 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; +import redis from 'lib/redis'; export async function getEventData(...args) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), + }).then(results => { + return Object.keys(results[0]).map(a => { + return { x: a, y: results[0][`${a}`] }; + }); }); } @@ -21,7 +26,7 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu on event.website_id = website.website_id join event_data on event.event_id = event_data.event_id - where website.website_id='${websiteId}' + where website.website_id ='${websiteId}' and event.created_at between $1 and $2 ${event_name ? `and event_name = ${event_name}` : ''} ${ @@ -30,23 +35,21 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu : '' }`, params, - ).then(results => { - return Object.keys(results[0]).map(a => { - return { x: a, y: results[0][`${a}`] }; - }); - }); + ); } async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) { const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } = clickhouse; - const params = [websiteId]; + const website = await redis.get(`website:${websiteId}`); + const params = [websiteId, website?.revId || 0]; return rawQuery( `select ${getEventDataColumnsQuery('event_data', columns)} from event - where website_id= $1 + where website_id = $1 + and rev_id = $2 ${event_name ? `and event_name = ${event_name}` : ''} and ${getBetweenDates('created_at', startDate, endDate)} ${ @@ -55,9 +58,5 @@ async function clickhouseQuery(websiteId, { startDate, endDate, event_name, colu : '' }`, params, - ).then(results => { - return Object.keys(results[0]).map(a => { - return { x: a, y: results[0][`${a}`] }; - }); - }); + ); } diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js index 9dcf6d55..8b6c8263 100644 --- a/queries/analytics/event/getEventMetrics.js +++ b/queries/analytics/event/getEventMetrics.js @@ -1,6 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import redis from 'lib/redis'; export async function getEventMetrics(...args) { return runQuery({ @@ -46,7 +47,8 @@ async function clickhouseQuery( filters = {}, ) { const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse; - const params = [websiteId]; + const website = await redis.get(`website:${websiteId}`); + const params = [websiteId, website?.revId || 0]; return rawQuery( `select @@ -55,7 +57,8 @@ async function clickhouseQuery( count(*) y from event where event_name != '' - and website_id= $1 + and website_id = $1 + anbd rev_id = $2 and ${getBetweenDates('created_at', start_at, end_at)} ${getFilterQuery('event', filters, params)} group by x, t diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js index a6bbe2fa..c6d3c827 100644 --- a/queries/analytics/event/saveEvent.js +++ b/queries/analytics/event/saveEvent.js @@ -3,6 +3,7 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; import { uuid } from 'lib/crypto'; +import redis from 'lib/redis'; export async function saveEvent(...args) { return runQuery({ @@ -43,11 +44,13 @@ async function clickhouseQuery( { session: { country, sessionUuid, ...sessionArgs }, eventUuid, url, eventName, eventData }, ) { const { getDateFormat, sendMessage } = kafka; + const website = await redis.get(`website:${websiteId}`); const params = { session_id: sessionUuid, event_id: eventUuid, website_id: websiteId, + rev_id: website?.revId || 0, created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), event_name: eventName?.substring(0, EVENT_NAME_LENGTH), diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js index 3acfdefb..62cda165 100644 --- a/queries/analytics/pageview/getPageviewMetrics.js +++ b/queries/analytics/pageview/getPageviewMetrics.js @@ -1,6 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import redis from 'lib/redis'; export async function getPageviewMetrics(...args) { return runQuery({ @@ -37,13 +38,15 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f async function clickhouseQuery(websiteId, { startDate, endDate, column, filters = {} }) { const { rawQuery, parseFilters, getBetweenDates } = clickhouse; - const params = [websiteId]; + const website = await redis.get(`website:${websiteId}`); + const params = [websiteId, website?.revId || 0]; const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params); return rawQuery( `select ${column} x, count(*) y from event - where website_id= $1 + where website_id = $1 + and rev_id = $2 ${column !== 'event_name' ? `and event_name = ''` : `and event_name != ''`} and ${getBetweenDates('created_at', startDate, endDate)} ${pageviewQuery} diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js index 57ff27f3..b18a3acc 100644 --- a/queries/analytics/pageview/getPageviewStats.js +++ b/queries/analytics/pageview/getPageviewStats.js @@ -1,6 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import redis from 'lib/redis'; export async function getPageviewStats(...args) { return runQuery({ @@ -51,7 +52,8 @@ async function clickhouseQuery( { start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} }, ) { const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse; - const params = [websiteId]; + const website = await redis.get(`website:${websiteId}`); + const params = [websiteId, website?.revId || 0]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js index c035bfdf..0763ab18 100644 --- a/queries/analytics/pageview/savePageView.js +++ b/queries/analytics/pageview/savePageView.js @@ -2,6 +2,7 @@ import { URL_LENGTH } from 'lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; +import redis from 'lib/redis'; export async function savePageView(...args) { return runQuery({ @@ -29,6 +30,7 @@ async function clickhouseQuery( websiteId, { session: { country, id: sessionId, ...sessionArgs }, url, referrer }, ) { + const website = await redis.get(`website:${websiteId}`); const { getDateFormat, sendMessage } = kafka; const params = { session_id: sessionId, @@ -36,6 +38,7 @@ async function clickhouseQuery( created_at: getDateFormat(new Date()), url: url?.substring(0, URL_LENGTH), referrer: referrer?.substring(0, URL_LENGTH), + rev_id: website?.revId || 0, ...sessionArgs, country: country ? country : null, }; diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js index b121ef7c..dfa8567b 100644 --- a/queries/analytics/session/createSession.js +++ b/queries/analytics/session/createSession.js @@ -7,34 +7,32 @@ export async function createSession(...args) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), + }).then(async data => { + if (redis.enabled && data) { + await redis.set(`session:${data.id}`, data); + } + + return data; }); } async function relationalQuery(websiteId, data) { - return prisma.client.session - .create({ - data: { - websiteId, - ...data, - }, - select: { - id: true, - hostname: true, - browser: true, - os: true, - screen: true, - language: true, - country: true, - device: true, - }, - }) - .then(async res => { - if (redis.enabled && res) { - await redis.set(`session:${res.id}`, 1); - } - - return res; - }); + return prisma.client.session.create({ + data: { + websiteId, + ...data, + }, + select: { + id: true, + hostname: true, + browser: true, + os: true, + screen: true, + language: true, + country: true, + device: true, + }, + }); } async function clickhouseQuery( @@ -42,10 +40,12 @@ async function clickhouseQuery( { sessionId, hostname, browser, os, screen, language, country, device }, ) { const { getDateFormat, sendMessage } = kafka; + const website = await redis.get(`website:${websiteId}`); - const params = { + const data = { sessionId, website_id: websiteId, + rev_id: website?.revId || 0, created_at: getDateFormat(new Date()), hostname, browser, @@ -56,9 +56,7 @@ async function clickhouseQuery( country: country ? country : null, }; - await sendMessage(params, 'event'); + await sendMessage(data, 'event'); - if (redis.enabled) { - await redis.set(`session:${sessionId}`, 1); - } + return data; } diff --git a/queries/analytics/session/getSession.js b/queries/analytics/session/getSession.js index 6bb4e445..36269fc0 100644 --- a/queries/analytics/session/getSession.js +++ b/queries/analytics/session/getSession.js @@ -7,26 +7,24 @@ export async function getSession(...args) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), + }).then(async data => { + if (redis.enabled && data) { + await redis.set(`session:${data.id}`, data); + } + + return data; }); } async function relationalQuery(where) { - return prisma.client.session - .findUnique({ - where, - }) - .then(async res => { - if (redis.enabled && res) { - await redis.set(`session:${res.sessionUuid}`, 1); - } - - return res; - }); + return prisma.client.session.findUnique({ + where, + }); } -async function clickhouseQuery(sessionUuid) { +async function clickhouseQuery(sessionId) { const { rawQuery, findFirst } = clickhouse; - const params = [sessionUuid]; + const params = [sessionId]; return rawQuery( `select distinct @@ -43,13 +41,5 @@ async function clickhouseQuery(sessionUuid) { from event where session_id = $1`, params, - ) - .then(result => findFirst(result)) - .then(async res => { - if (redis.enabled && res) { - await redis.set(`session:${res.id}`, 1); - } - - return res; - }); + ).then(result => findFirst(result)); } diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js index e4e389d0..797230f8 100644 --- a/queries/analytics/session/getSessionMetrics.js +++ b/queries/analytics/session/getSessionMetrics.js @@ -1,6 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; +import redis from 'lib/redis'; export async function getSessionMetrics(...args) { return runQuery({ @@ -36,13 +37,15 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters = async function clickhouseQuery(websiteId, { startDate, endDate, field, filters = {} }) { const { parseFilters, getBetweenDates, rawQuery } = clickhouse; - const params = [websiteId]; + const website = await redis.get(`website:${websiteId}`); + const params = [websiteId, website?.revId || 0]; const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params); return rawQuery( `select ${field} x, count(*) y from event as x - where website_id=$1 + where website_id = $1 + and rev_id = $2 and event_name = '' and ${getBetweenDates('created_at', startDate, endDate)} ${pageviewQuery}