diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.js index 6c14c2c1..eda93f0b 100644 --- a/components/pages/console/TestConsole.js +++ b/components/pages/console/TestConsole.js @@ -28,22 +28,41 @@ export function TestConsole() { window.umami.track({ url: '/page-view', referrer: 'https://www.google.com' }); window.umami.track('track-event-no-data'); window.umami.track('track-event-with-data', { - data: { + test: 'test-data', + boolean: true, + booleanError: 'true', + time: new Date(), + number: 1, + time2: new Date().toISOString(), + nested: { test: 'test-data', - boolean: true, - booleanError: 'true', - time: new Date(), number: 1, - time2: new Date().toISOString(), - nested: { + object: { test: 'test-data', - number: 1, - object: { - test: 'test-data', - }, }, - array: [1, 2, 3], }, + array: [1, 2, 3], + }); + } + + function handleIdentifyClick() { + window.umami.identify({ + userId: 123, + name: 'brian', + number: Math.random() * 100, + test: 'test-data', + boolean: true, + booleanError: 'true', + time: new Date(), + time2: new Date().toISOString(), + nested: { + test: 'test-data', + number: 1, + object: { + test: 'test-data', + }, + }, + array: [1, 2, 3], }); } @@ -116,6 +135,10 @@ export function TestConsole() { +

+ diff --git a/db/clickhouse/migrations/01_edit_keys.sql b/db/clickhouse/migrations/01_edit_keys.sql new file mode 100644 index 00000000..b948ff38 --- /dev/null +++ b/db/clickhouse/migrations/01_edit_keys.sql @@ -0,0 +1,4 @@ +ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value"; +ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value"; +ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value"; +ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type"; \ No newline at end of file diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 8f48b434..de1082ec 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -117,10 +117,10 @@ CREATE TABLE umami.event_data url_path String, event_name String, event_key String, - event_string_value Nullable(String), - event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 - event_date_value Nullable(DateTime('UTC')), - event_data_type UInt32, + string_value Nullable(String), + numeric_value Nullable(Decimal64(4)), --922337203685477.5625 + date_value Nullable(DateTime('UTC')), + data_type UInt32, created_at DateTime('UTC') ) engine = MergeTree @@ -134,10 +134,10 @@ CREATE TABLE umami.event_data_queue ( url_path String, event_name String, event_key String, - event_string_value Nullable(String), - event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625 - event_date_value Nullable(DateTime('UTC')), - event_data_type UInt32, + string_value Nullable(String), + numeric_value Nullable(Decimal64(4)), --922337203685477.5625 + date_value Nullable(DateTime('UTC')), + data_type UInt32, created_at DateTime('UTC'), --virtual columns _error String, @@ -158,10 +158,10 @@ SELECT website_id, url_path, event_name, event_key, - event_string_value, - event_numeric_value, - event_date_value, - event_data_type, + string_value, + numeric_value, + date_value, + data_type, created_at FROM umami.event_data_queue; diff --git a/db/mysql/migrations/03_session_data/migration.sql b/db/mysql/migrations/03_session_data/migration.sql new file mode 100644 index 00000000..3618080d --- /dev/null +++ b/db/mysql/migrations/03_session_data/migration.sql @@ -0,0 +1,37 @@ +/* + Warnings: + + - The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint. + - You are about to drop the column `event_data_type` on the `event_data` table. All the data in the column will be lost. + - You are about to drop the column `event_date_value` on the `event_data` table. All the data in the column will be lost. + - You are about to drop the column `event_id` on the `event_data` table. All the data in the column will be lost. + - You are about to drop the column `event_numeric_value` on the `event_data` table. All the data in the column will be lost. + - You are about to drop the column `event_string_value` on the `event_data` table. All the data in the column will be lost. + - Added the required column `data_type` to the `event_data` table without a default value. This is not possible if the table is not empty. + - Added the required column `event_data_id` to the `event_data` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE `event_data` RENAME COLUMN `event_data_type` TO `data_type`; +ALTER TABLE `event_data` RENAME COLUMN `event_date_value` TO `date_value`; +ALTER TABLE `event_data` RENAME COLUMN `event_id` TO `event_data_id`; +ALTER TABLE `event_data` RENAME COLUMN `event_numeric_value` TO `numeric_value`; +ALTER TABLE `event_data` RENAME COLUMN `event_string_value` TO `string_value`; + +-- CreateTable +CREATE TABLE `session_data` ( + `session_data_id` VARCHAR(36) NOT NULL, + `website_id` VARCHAR(36) NOT NULL, + `session_id` VARCHAR(36) NOT NULL, + `event_key` VARCHAR(500) NOT NULL, + `event_string_value` VARCHAR(500) NULL, + `event_numeric_value` DECIMAL(19, 4) NULL, + `event_date_value` TIMESTAMP(0) NULL, + `event_data_type` INTEGER UNSIGNED NOT NULL, + `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), + + INDEX `session_data_created_at_idx`(`created_at`), + INDEX `session_data_website_id_idx`(`website_id`), + INDEX `session_data_session_id_idx`(`session_id`), + PRIMARY KEY (`session_data_id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 0752f418..88d5ef80 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -14,12 +14,12 @@ model User { password String @db.VarChar(60) role String @map("role") @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) - updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0) + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0) deletedAt DateTime? @map("deleted_at") @db.Timestamp(0) website Website[] teamUser TeamUser[] - Report Report[] + report Report[] @@map("user") } @@ -40,6 +40,7 @@ model Session { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) websiteEvent WebsiteEvent[] + sessionData SessionData[] @@index([createdAt]) @@index([websiteId]) @@ -54,13 +55,14 @@ model Website { resetAt DateTime? @map("reset_at") @db.Timestamp(0) userId String? @map("user_id") @db.VarChar(36) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) - updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0) + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0) deletedAt DateTime? @map("deleted_at") @db.Timestamp(0) user User? @relation(fields: [userId], references: [id]) teamWebsite TeamWebsite[] eventData EventData[] - Report Report[] + report Report[] + sessionData SessionData[] @@index([userId]) @@index([createdAt]) @@ -94,15 +96,15 @@ model WebsiteEvent { } model EventData { - id String @id() @map("event_id") @db.VarChar(36) - websiteEventId String @map("website_event_id") @db.VarChar(36) - websiteId String @map("website_id") @db.VarChar(36) - eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String? @map("event_string_value") @db.VarChar(500) - eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) - eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0) - eventDataType Int @map("event_data_type") @db.UnsignedInt - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) + id String @id() @map("event_data_id") @db.VarChar(36) + websiteEventId String @map("website_event_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + eventKey String @map("event_key") @db.VarChar(500) + stringValue String? @map("string_value") @db.VarChar(500) + numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4) + dateValue DateTime? @map("date_value") @db.Timestamp(0) + dataType Int @map("data_type") @db.UnsignedInt + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) @@ -114,12 +116,32 @@ model EventData { @@map("event_data") } +model SessionData { + id String @id() @map("session_data_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + sessionId String @map("session_id") @db.VarChar(36) + eventKey String @map("event_key") @db.VarChar(500) + eventStringValue String? @map("event_string_value") @db.VarChar(500) + eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) + eventDateValue DateTime? @map("event_date_value") @db.Timestamp(0) + eventDataType Int @map("event_data_type") @db.UnsignedInt + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) + + website Website @relation(fields: [websiteId], references: [id]) + session Session @relation(fields: [sessionId], references: [id]) + + @@index([createdAt]) + @@index([websiteId]) + @@index([sessionId]) + @@map("session_data") +} + model Team { id String @id() @unique() @map("team_id") @db.VarChar(36) name String @db.VarChar(50) accessCode String? @unique @map("access_code") @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) - updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0) + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0) teamUser TeamUser[] teamWebsite TeamWebsite[] @@ -134,7 +156,7 @@ model TeamUser { userId String @map("user_id") @db.VarChar(36) role String @map("role") @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) - updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0) + updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0) team Team @relation(fields: [teamId], references: [id]) user User @relation(fields: [userId], references: [id]) @@ -177,4 +199,4 @@ model Report { @@index([type]) @@index([name]) @@map("report") -} \ No newline at end of file +} diff --git a/db/postgresql/migrations/03_session_data/migration.sql b/db/postgresql/migrations/03_session_data/migration.sql new file mode 100644 index 00000000..233c3793 --- /dev/null +++ b/db/postgresql/migrations/03_session_data/migration.sql @@ -0,0 +1,31 @@ +-- AlterTable +ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type"; +ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value"; +ALTER TABLE "event_data" RENAME COLUMN "event_id" TO "event_data_id"; +ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value"; +ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value"; + +-- CreateTable +CREATE TABLE "session_data" ( + "session_data_id" UUID NOT NULL, + "website_id" UUID NOT NULL, + "session_id" UUID NOT NULL, + "session_key" VARCHAR(500) NOT NULL, + "string_value" VARCHAR(500), + "numeric_value" DECIMAL(19,4), + "date_value" TIMESTAMPTZ(6), + "data_type" INTEGER NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + "deleted_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "session_data_pkey" PRIMARY KEY ("session_data_id") +); + +-- CreateIndex +CREATE INDEX "session_data_created_at_idx" ON "session_data"("created_at"); + +-- CreateIndex +CREATE INDEX "session_data_website_id_idx" ON "session_data"("website_id"); + +-- CreateIndex +CREATE INDEX "session_data_session_id_idx" ON "session_data"("session_id"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index 7c4cb94f..bfd71fd1 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -19,7 +19,7 @@ model User { website Website[] teamUser TeamUser[] - Report Report[] + report Report[] @@map("user") } @@ -40,6 +40,7 @@ model Session { createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) websiteEvent WebsiteEvent[] + sessionData SessionData[] @@index([createdAt]) @@index([websiteId]) @@ -60,7 +61,8 @@ model Website { user User? @relation(fields: [userId], references: [id]) teamWebsite TeamWebsite[] eventData EventData[] - Report Report[] + report Report[] + sessionData SessionData[] @@index([userId]) @@index([createdAt]) @@ -94,15 +96,15 @@ model WebsiteEvent { } model EventData { - id String @id() @map("event_id") @db.Uuid - websiteId String @map("website_id") @db.Uuid - websiteEventId String @map("website_event_id") @db.Uuid - eventKey String @map("event_key") @db.VarChar(500) - eventStringValue String? @map("event_string_value") @db.VarChar(500) - eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4) - eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6) - eventDataType Int @map("event_data_type") @db.Integer - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + id String @id() @map("event_data_id") @db.Uuid + websiteId String @map("website_id") @db.Uuid + websiteEventId String @map("website_event_id") @db.Uuid + eventKey String @map("event_key") @db.VarChar(500) + stringValue String? @map("string_value") @db.VarChar(500) + numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4) + dateValue DateTime? @map("date_value") @db.Timestamptz(6) + dataType Int @map("data_type") @db.Integer + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id]) websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id]) @@ -113,6 +115,27 @@ model EventData { @@map("event_data") } +model SessionData { + id String @id() @map("session_data_id") @db.Uuid + websiteId String @map("website_id") @db.Uuid + sessionId String @map("session_id") @db.Uuid + sessionKey String @map("session_key") @db.VarChar(500) + stringValue String? @map("string_value") @db.VarChar(500) + numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4) + dateValue DateTime? @map("date_value") @db.Timestamptz(6) + dataType Int @map("data_type") @db.Integer + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + deletedAt DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6) + + website Website @relation(fields: [websiteId], references: [id]) + session Session @relation(fields: [sessionId], references: [id]) + + @@index([createdAt]) + @@index([websiteId]) + @@index([sessionId]) + @@map("session_data") +} + model Team { id String @id() @unique() @map("team_id") @db.Uuid name String @db.VarChar(50) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index e97be806..c91cf3da 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse'; import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; -import { getEventDataType } from './eventData'; +import { getDynamicDataType } from './dynamicData'; import { WebsiteMetricFilter } from './types'; import { FILTER_COLUMNS } from './constants'; @@ -74,7 +74,7 @@ function getEventDataFilterQuery( params: any, ) { const query = filters.reduce((ac, cv, i) => { - const type = getEventDataType(cv.eventValue); + const type = getDynamicDataType(cv.eventValue); let value = cv.eventValue; diff --git a/lib/constants.ts b/lib/constants.ts index 7eef10b1..9edbe100 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -42,6 +42,11 @@ export const SESSION_COLUMNS = [ 'city', ]; +export const COLLECTION_TYPE = { + event: 'event', + identify: 'identify', +}; + export const FILTER_COLUMNS = { url: 'url_path', referrer: 'referrer_domain', @@ -56,7 +61,7 @@ export const EVENT_TYPE = { customEvent: 2, } as const; -export const EVENT_DATA_TYPE = { +export const DYNAMIC_DATA_TYPE = { string: 1, number: 2, boolean: 3, diff --git a/lib/eventData.ts b/lib/dynamicData.ts similarity index 63% rename from lib/eventData.ts rename to lib/dynamicData.ts index aee1f9b4..da8eb8b2 100644 --- a/lib/eventData.ts +++ b/lib/dynamicData.ts @@ -1,12 +1,12 @@ import { isValid, parseISO } from 'date-fns'; -import { EVENT_DATA_TYPE } from './constants'; -import { EventDataTypes } from './types'; +import { DYNAMIC_DATA_TYPE } from './constants'; +import { DynamicDataType } from './types'; export function flattenJSON( eventData: { [key: string]: any }, - keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [], + keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [], parentKey = '', -): { key: string; value: any; eventDataType: EventDataTypes }[] { +): { key: string; value: any; dynamicDataType: DynamicDataType }[] { return Object.keys(eventData).reduce( (acc, key) => { const value = eventData[key]; @@ -25,7 +25,7 @@ export function flattenJSON( ).keyValues; } -export function getEventDataType(value: any): string { +export function getDynamicDataType(value: any): string { let type: string = typeof value; if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) { @@ -36,34 +36,34 @@ export function getEventDataType(value: any): string { } function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) { - const type = getEventDataType(value); + const type = getDynamicDataType(value); - let eventDataType = null; + let dynamicDataType = null; switch (type) { case 'number': - eventDataType = EVENT_DATA_TYPE.number; + dynamicDataType = DYNAMIC_DATA_TYPE.number; break; case 'string': - eventDataType = EVENT_DATA_TYPE.string; + dynamicDataType = DYNAMIC_DATA_TYPE.string; break; case 'boolean': - eventDataType = EVENT_DATA_TYPE.boolean; + dynamicDataType = DYNAMIC_DATA_TYPE.boolean; value = value ? 'true' : 'false'; break; case 'date': - eventDataType = EVENT_DATA_TYPE.date; + dynamicDataType = DYNAMIC_DATA_TYPE.date; break; case 'object': - eventDataType = EVENT_DATA_TYPE.array; + dynamicDataType = DYNAMIC_DATA_TYPE.array; value = JSON.stringify(value); break; default: - eventDataType = EVENT_DATA_TYPE.string; + dynamicDataType = DYNAMIC_DATA_TYPE.string; break; } - acc.keyValues.push({ key, value, eventDataType }); + acc.keyValues.push({ key, value, dynamicDataType }); } function getKeyName(key, parentKey) { diff --git a/lib/prisma.ts b/lib/prisma.ts index fdd8a58d..51707049 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -1,7 +1,7 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; -import { getEventDataType } from './eventData'; +import { getDynamicDataType } from './dynamicData'; import { FILTER_COLUMNS } from './constants'; const MYSQL_DATE_FORMATS = { @@ -85,7 +85,7 @@ function getEventDataFilterQuery( params: any[], ) { const query = filters.reduce((ac, cv) => { - const type = getEventDataType(cv.eventValue); + const type = getDynamicDataType(cv.eventValue); let value = cv.eventValue; diff --git a/lib/session.ts b/lib/session.ts index 1fedb91b..04cbc9b0 100644 --- a/lib/session.ts +++ b/lib/session.ts @@ -1,12 +1,11 @@ -import clickhouse from 'lib/clickhouse'; import { secret, uuid } from 'lib/crypto'; import { getClientInfo, getJsonBody } from 'lib/detect'; import { parseToken } from 'next-basics'; import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send'; import { createSession } from 'queries'; import { validate } from 'uuid'; -import { loadSession, loadWebsite } from './query'; import cache from './cache'; +import { loadSession, loadWebsite } from './query'; export async function findSession(req: NextApiRequestCollect) { const { payload } = getJsonBody(req); @@ -46,26 +45,8 @@ export async function findSession(req: NextApiRequestCollect) { const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } = await getClientInfo(req, payload); - const sessionId = uuid(websiteId, hostname, ip, userAgent); - // Clickhouse does not require session lookup - if (clickhouse.enabled) { - return { - id: sessionId, - websiteId, - hostname, - browser, - os, - device, - screen, - language, - country, - subdivision1, - subdivision2, - city, - ownerId: website.userId, - }; - } + const sessionId = uuid(websiteId, hostname, ip, userAgent); // Find session let session = await loadSession(sessionId); diff --git a/lib/types.ts b/lib/types.ts index 37c1ffdc..05c09120 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,18 +1,20 @@ import { NextApiRequest } from 'next'; -import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants'; +import { COLLECTION_TYPE, DYNAMIC_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants'; type ObjectValues = T[keyof T]; -export type Roles = ObjectValues; +export type CollectionType = ObjectValues; -export type EventTypes = ObjectValues; +export type Role = ObjectValues; -export type EventDataTypes = ObjectValues; +export type EventType = ObjectValues; -export type KafkaTopics = ObjectValues; +export type DynamicDataType = ObjectValues; -export interface EventData { - [key: string]: number | string | EventData | number[] | string[] | EventData[]; +export type KafkaTopic = ObjectValues; + +export interface DynamicData { + [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[]; } export interface Auth { diff --git a/pages/api/send.ts b/pages/api/send.ts index df7ceb6e..24264fa3 100644 --- a/pages/api/send.ts +++ b/pages/api/send.ts @@ -7,6 +7,9 @@ import { getJsonBody, getIpAddress } from 'lib/detect'; import { secret } from 'lib/crypto'; import { NextApiRequest, NextApiResponse } from 'next'; import { Resolver } from 'dns/promises'; +import { CollectionType } from 'lib/types'; +import { COLLECTION_TYPE } from 'lib/constants'; +import { saveSessionData } from 'queries/analytics/session/saveSessionData'; export interface CollectRequestBody { payload: { @@ -20,7 +23,7 @@ export interface CollectRequestBody { website: string; name: string; }; - type: string; + type: CollectionType; } export interface NextApiRequestCollect extends NextApiRequest { @@ -52,17 +55,81 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { const { type, payload } = getJsonBody(req); - if (type !== 'event') { + validateBody(res, { type, payload }); + + if (await hasBlockedIp(req)) { + return forbidden(res); + } + + const { url, referrer, name: eventName, data: dynamicData, title: pageTitle } = payload; + + await useSession(req, res); + + const session = req.session; + + if (type === COLLECTION_TYPE.event) { + // eslint-disable-next-line prefer-const + let [urlPath, urlQuery] = url?.split('?') || []; + let [referrerPath, referrerQuery] = referrer?.split('?') || []; + let referrerDomain; + + if (!urlPath) { + urlPath = '/'; + } + + if (referrerPath?.startsWith('http')) { + const refUrl = new URL(referrer); + referrerPath = refUrl.pathname; + referrerQuery = refUrl.search.substring(1); + referrerDomain = refUrl.hostname.replace(/www\./, ''); + } + + if (process.env.REMOVE_TRAILING_SLASH) { + urlPath = urlPath.replace(/.+\/$/, ''); + } + + await saveEvent({ + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + pageTitle, + eventName, + eventData: dynamicData, + ...session, + sessionId: session.id, + }); + } + + if (type === COLLECTION_TYPE.identify) { + if (!dynamicData) { + return badRequest(res, 'Data required.'); + } + + await saveSessionData({ ...session, sessionData: dynamicData, sessionId: session.id }); + } + + const token = createToken(session, secret()); + + return send(res, token); +}; + +function validateBody(res: NextApiResponse, { type, payload }: CollectRequestBody) { + const { data } = payload; + + // Validate type + if (type !== COLLECTION_TYPE.event && type !== COLLECTION_TYPE.identify) { return badRequest(res, 'Wrong payload type.'); } - const { url, referrer, name: eventName, data: eventData, title: pageTitle } = payload; - // Validate eventData is JSON - if (eventData && !(typeof eventData === 'object' && !Array.isArray(eventData))) { + if (data && !(typeof data === 'object' && !Array.isArray(data))) { return badRequest(res, 'Invalid event data.'); } +} +async function hasBlockedIp(req: NextApiRequestCollect) { const ignoreIps = process.env.IGNORE_IP; const ignoreHostnames = process.env.IGNORE_HOSTNAME; @@ -100,49 +167,6 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { return false; }); - if (blocked) { - return forbidden(res); - } + return blocked; } - - await useSession(req, res); - - const session = req.session; - - // eslint-disable-next-line prefer-const - let [urlPath, urlQuery] = url?.split('?') || []; - let [referrerPath, referrerQuery] = referrer?.split('?') || []; - let referrerDomain; - - if (!urlPath) { - urlPath = '/'; - } - - if (referrerPath?.startsWith('http')) { - const refUrl = new URL(referrer); - referrerPath = refUrl.pathname; - referrerQuery = refUrl.search.substring(1); - referrerDomain = refUrl.hostname.replace(/www\./, ''); - } - - if (process.env.REMOVE_TRAILING_SLASH) { - urlPath = urlPath.replace(/.+\/$/, ''); - } - - await saveEvent({ - urlPath, - urlQuery, - referrerPath, - referrerQuery, - referrerDomain, - pageTitle, - eventName, - eventData, - ...session, - sessionId: session.id, - }); - - const token = createToken(session, secret()); - - return send(res, token); -}; +} diff --git a/pages/api/users/[id]/index.ts b/pages/api/users/[id]/index.ts index de4642cb..a4ab05ff 100644 --- a/pages/api/users/[id]/index.ts +++ b/pages/api/users/[id]/index.ts @@ -1,4 +1,4 @@ -import { NextApiRequestQueryBody, Roles, User } from 'lib/types'; +import { NextApiRequestQueryBody, Role, User } from 'lib/types'; import { canDeleteUser, canUpdateUser, canViewUser } from 'lib/auth'; import { useAuth } from 'lib/middleware'; import { NextApiResponse } from 'next'; @@ -12,7 +12,7 @@ export interface UserRequestQuery { export interface UserRequestBody { username: string; password: string; - role: Roles; + role: Role; } export default async ( diff --git a/pages/api/users/index.ts b/pages/api/users/index.ts index 4d35d856..c6103c35 100644 --- a/pages/api/users/index.ts +++ b/pages/api/users/index.ts @@ -2,7 +2,7 @@ import { canCreateUser, canViewUsers } from 'lib/auth'; import { ROLES } from 'lib/constants'; import { uuid } from 'lib/crypto'; import { useAuth } from 'lib/middleware'; -import { NextApiRequestQueryBody, Roles, User } from 'lib/types'; +import { NextApiRequestQueryBody, Role, User } from 'lib/types'; import { NextApiResponse } from 'next'; import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics'; import { createUser, getUser, getUsers } from 'queries'; @@ -11,7 +11,7 @@ export interface UsersRequestBody { username: string; password: string; id: string; - role?: Roles; + role?: Role; } export default async ( diff --git a/queries/admin/user.ts b/queries/admin/user.ts index 3ba2654b..ec23559f 100644 --- a/queries/admin/user.ts +++ b/queries/admin/user.ts @@ -3,7 +3,7 @@ import { getRandomChars } from 'next-basics'; import cache from 'lib/cache'; import { ROLES } from 'lib/constants'; import prisma from 'lib/prisma'; -import { Website, User, Roles } from 'lib/types'; +import { Website, User, Role } from 'lib/types'; export async function getUser( where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput, @@ -91,7 +91,7 @@ export async function createUser(data: { id: string; username: string; password: string; - role: Roles; + role: Role; }): Promise<{ id: string; username: string; diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.ts index 9a7db00d..51087a59 100644 --- a/queries/analytics/event/saveEvent.ts +++ b/queries/analytics/event/saveEvent.ts @@ -133,9 +133,10 @@ async function clickhouseQuery(data: { const createdAt = getDateFormat(new Date()); const message = { + ...args, website_id: websiteId, session_id: sessionId, - event_id: eventId, + event_id: uuid(), country: country ? country : null, subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null, subdivision2: subdivision2 ? subdivision2 : null, @@ -149,7 +150,6 @@ async function clickhouseQuery(data: { event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, created_at: createdAt, - ...args, }; await sendMessage(message, 'event'); diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index 90e63565..96ea8831 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -1,11 +1,11 @@ import { Prisma } from '@prisma/client'; -import { EVENT_DATA_TYPE } from 'lib/constants'; +import { DYNAMIC_DATA_TYPE } from 'lib/constants'; import { uuid } from 'lib/crypto'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { flattenJSON } from 'lib/eventData'; +import { flattenJSON } from 'lib/dynamicData'; import kafka from 'lib/kafka'; import prisma from 'lib/prisma'; -import { EventData } from 'lib/types'; +import { DynamicData } from 'lib/types'; export async function saveEventData(args: { websiteId: string; @@ -13,7 +13,7 @@ export async function saveEventData(args: { sessionId?: string; urlPath?: string; eventName?: string; - eventData: EventData; + eventData: DynamicData; createdAt?: string; }) { return runQuery({ @@ -25,7 +25,7 @@ export async function saveEventData(args: { async function relationalQuery(data: { websiteId: string; eventId: string; - eventData: EventData; + eventData: DynamicData; }): Promise { const { websiteId, eventId, eventData } = data; @@ -36,16 +36,16 @@ async function relationalQuery(data: { id: uuid(), websiteEventId: eventId, websiteId, - eventKey: a.key, - eventStringValue: - a.eventDataType === EVENT_DATA_TYPE.string || - a.eventDataType === EVENT_DATA_TYPE.boolean || - a.eventDataType === EVENT_DATA_TYPE.array + key: a.key, + stringValue: + a.dynamicDataType === DYNAMIC_DATA_TYPE.string || + a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean || + a.dynamicDataType === DYNAMIC_DATA_TYPE.array ? a.value : null, - eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null, - eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null, - eventDataType: a.eventDataType, + numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null, + dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dynamicDataType, })); return prisma.client.eventData.createMany({ @@ -59,7 +59,7 @@ async function clickhouseQuery(data: { sessionId?: string; urlPath?: string; eventName?: string; - eventData: EventData; + eventData: DynamicData; createdAt?: string; }) { const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data; @@ -75,15 +75,15 @@ async function clickhouseQuery(data: { url_path: urlPath, event_name: eventName, event_key: a.key, - event_string_value: - a.eventDataType === EVENT_DATA_TYPE.string || - a.eventDataType === EVENT_DATA_TYPE.boolean || - a.eventDataType === EVENT_DATA_TYPE.array + string_value: + a.dynamicDataType === DYNAMIC_DATA_TYPE.string || + a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean || + a.dynamicDataType === DYNAMIC_DATA_TYPE.array ? a.value : null, - event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null, - event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null, - event_data_type: a.eventDataType, + numeric_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null, + date_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? getDateFormat(a.value) : null, + data_type: a.dynamicDataType, created_at: createdAt, })); diff --git a/queries/analytics/session/createSession.ts b/queries/analytics/session/createSession.ts index 22f7892f..4fd36d2e 100644 --- a/queries/analytics/session/createSession.ts +++ b/queries/analytics/session/createSession.ts @@ -1,23 +1,8 @@ -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import kafka from 'lib/kafka'; -import prisma from 'lib/prisma'; -import cache from 'lib/cache'; import { Prisma } from '@prisma/client'; +import cache from 'lib/cache'; +import prisma from 'lib/prisma'; -export async function createSession(args: Prisma.SessionCreateInput) { - return runQuery({ - [PRISMA]: () => relationalQuery(args), - [CLICKHOUSE]: () => clickhouseQuery(args), - }).then(async data => { - if (cache.enabled) { - await cache.storeSession(data); - } - - return data; - }); -} - -async function relationalQuery(data: Prisma.SessionCreateInput) { +export async function createSession(data: Prisma.SessionCreateInput) { const { id, websiteId, @@ -33,71 +18,28 @@ async function relationalQuery(data: Prisma.SessionCreateInput) { city, } = data; - return prisma.client.session.create({ - data: { - id, - websiteId, - hostname, - browser, - os, - device, - screen, - language, - country, - subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null, - subdivision2, - city, - }, - }); -} - -async function clickhouseQuery(data: { - id: string; - websiteId: string; - hostname?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - subdivision1?: string; - subdivision2?: string; - city?: string; -}) { - const { - id, - websiteId, - hostname, - browser, - os, - device, - screen, - language, - country, - subdivision1, - subdivision2, - city, - } = data; - const { getDateFormat, sendMessage } = kafka; - - const msg = { - session_id: id, - website_id: websiteId, - hostname, - browser, - os, - device, - screen, - language, - country, - subdivision1, - subdivision2, - city, - created_at: getDateFormat(new Date()), - }; - - await sendMessage(msg, 'event'); - - return data; + return prisma.client.session + .create({ + data: { + id, + websiteId, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null, + subdivision2, + city, + }, + }) + .then(async data => { + if (cache.enabled) { + await cache.storeSession(data); + } + + return data; + }); } diff --git a/queries/analytics/session/getSession.ts b/queries/analytics/session/getSession.ts index d226e832..2fd8d18f 100644 --- a/queries/analytics/session/getSession.ts +++ b/queries/analytics/session/getSession.ts @@ -1,43 +1,8 @@ -import clickhouse from 'lib/clickhouse'; -import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import prisma from 'lib/prisma'; import { Prisma } from '@prisma/client'; +import prisma from 'lib/prisma'; -export async function getSession(args: { id: string }) { - return runQuery({ - [PRISMA]: () => relationalQuery(args), - [CLICKHOUSE]: () => clickhouseQuery(args), - }); -} - -async function relationalQuery(where: Prisma.SessionWhereUniqueInput) { +export async function getSession(where: Prisma.SessionWhereUniqueInput) { return prisma.client.session.findUnique({ where, }); } - -async function clickhouseQuery({ id: sessionId }: { id: string }) { - const { rawQuery, findFirst } = clickhouse; - const params = { sessionId }; - - return rawQuery( - `select - session_id, - website_id, - created_at, - hostname, - browser, - os, - device, - screen, - language, - country, - subdivision1, - subdivision2, - city - from website_event - where session_id = {sessionId:UUID} - limit 1`, - params, - ).then(result => findFirst(result)); -} diff --git a/queries/analytics/session/saveSessionData.ts b/queries/analytics/session/saveSessionData.ts new file mode 100644 index 00000000..76842b4f --- /dev/null +++ b/queries/analytics/session/saveSessionData.ts @@ -0,0 +1,43 @@ +import { DYNAMIC_DATA_TYPE } from 'lib/constants'; +import { uuid } from 'lib/crypto'; +import { flattenJSON } from 'lib/dynamicData'; +import prisma from 'lib/prisma'; +import { DynamicData } from 'lib/types'; + +export async function saveSessionData(data: { + websiteId: string; + sessionId: string; + sessionData: DynamicData; +}) { + const { client, transaction } = prisma; + const { websiteId, sessionId, sessionData } = data; + + const jsonKeys = flattenJSON(sessionData); + + const flattendData = jsonKeys.map(a => ({ + id: uuid(), + websiteId, + sessionId, + key: a.key, + stringValue: + a.dynamicDataType === DYNAMIC_DATA_TYPE.string || + a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean || + a.dynamicDataType === DYNAMIC_DATA_TYPE.array + ? a.value + : null, + numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null, + dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dynamicDataType, + })); + + return transaction([ + client.sessionData.deleteMany({ + where: { + sessionId, + }, + }), + client.sessionData.createMany({ + data: flattendData, + }), + ]); +} diff --git a/tracker/index.js b/tracker/index.js index 1c40036e..6127ed19 100644 --- a/tracker/index.js +++ b/tracker/index.js @@ -173,7 +173,7 @@ } }; - const send = payload => { + const send = (payload, type = 'event') => { if (trackingDisabled()) return; const headers = { 'Content-Type': 'application/json', @@ -183,7 +183,7 @@ } return fetch(endpoint, { method: 'POST', - body: JSON.stringify({ type: 'event', payload }), + body: JSON.stringify({ type, payload }), headers, }) .then(res => res.text()) @@ -205,11 +205,14 @@ return send(getPayload()); }; + const identify = data => send({ ...getPayload(), data }, 'identify'); + /* Start */ if (!window.umami) { window.umami = { track, + identify, }; }