ambiser
/
umami
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: sql to mongo pipeline

pull/2040/head
minkik 2023-05-18 09:10:35 +09:00 committed by Joseph Lee
parent d5fac29dac
commit ed29fc08e8
7 changed files with 558 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pages/api/websites/[id]/stats.ts
View File

@ -92,14 +92,22 @@ export default async (
city, city,
}, },
}); });
let stats: object = {
const stats = Object.keys(metrics[0]).reduce((obj, key) => { _id: '',
obj[key] = { pageviews: 0,
value: Number(metrics[0][key]) || 0, uniques: 0,
change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0, bounces: 0,
}; totaltime: 0,
return obj; };
}, {}); if (metrics.length != 0) {
stats = Object.keys(metrics[0]).reduce((obj, key) => {
obj[key] = {
value: Number(metrics[0][key]) || 0,
change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0,
};
return obj;
}, {});
}
return ok(res, stats); return ok(res, stats);
} }

104
queries/analytics/event/getEventMetrics.ts
View File

@ -45,14 +45,107 @@ async function relationalQuery(
}; };
}, },
) { ) {
const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma; const { getDatabaseType, toUuid, rawQuery, getDateQuery, getFilterQuery, client } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt); const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [websiteId, resetDate, startDate, endDate]; const params: any = [websiteId, resetDate, startDate, endDate];
const filterQuery = getFilterQuery(filters, params); const filterQuery = getFilterQuery(filters, params);
const db = getDatabaseType();
return rawQuery( if (db === 'mongodb') {
`select return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.customEvent],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$project: {
t: {
$dateTrunc: {
date: '$created_at',
unit: unit,
timezone: timezone,
},
},
event_name: 1,
},
},
{
$group: {
_id: {
t: '$t',
name: '$event_name',
},
y: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id.name',
t: {
$dateToString: {
date: '$_id.t',
format: getDateQuery('', unit, timezone),
timezone: timezone,
},
},
y: 1,
_id: 0,
},
},
{
$sort: {
t: 1,
},
},
],
});
} else {
return rawQuery(
`select
event_name x, event_name x,
${getDateQuery('created_at', unit, timezone)} t, ${getDateQuery('created_at', unit, timezone)} t,
count(*) y count(*) y
@ -64,8 +157,9 @@ async function relationalQuery(
${filterQuery} ${filterQuery}
group by 1, 2 group by 1, 2
order by 2`, order by 2`,
params, params,
); );
}
} }
async function clickhouseQuery( async function clickhouseQuery(

121
queries/analytics/eventData/getEventData.ts
View File

@ -47,13 +47,123 @@ async function relationalQuery(
}, },
) { ) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data; const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma; const { getDatabaseType, toUuid, rawQuery, getEventDataFilterQuery, getDateQuery, client } =
prisma;
const db = getDatabaseType();
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt); const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [websiteId, resetDate, startDate, endDate, eventName || '']; const params: any = [websiteId, resetDate, startDate, endDate, eventName || ''];
return rawQuery( if (db === 'mongodb') {
`select let joinAggregation: any = { match: {} };
let matchAggregation: any = { match: {} };
let eventTypeProjectProperty = '';
let urlProjectProperty = '';
if (eventName || urlPath) {
joinAggregation = {
$lookup: {
from: 'website_event',
localField: 'website_event_id',
foreignField: '_id',
as: 'result',
},
};
eventTypeProjectProperty = 'event_name: {$arrayElemAt: ["$result.event_name", 0]}';
}
if (eventName) {
matchAggregation = {
$match: {
'result.event_name': eventName,
},
};
}
if (urlPath) {
urlProjectProperty = 'url_path: {$arrayElemAt: ["$result.url_path", 0],}';
}
let timeProjectProperty = '';
if (timeSeries) {
timeProjectProperty = `t: $dateTrunc: {date: "$created_at",unit: ${timeSeries.unit}, timezone : ${timeSeries.timezone}`;
}
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
joinAggregation,
matchAggregation,
{
$project: {
eventTypeProjectProperty,
timeProjectProperty,
urlProjectProperty,
},
},
{
$group: {
_id: {
url_path: '$url_path',
event_name: '$event_name',
t: '$t',
},
x: {
$sum: 1,
},
},
},
{
$project: {
url_path: '$_id.url_path',
urlPath: '$_id.url_path',
event_name: '$_id.event_name',
eventName: '$_id.event_name',
x: 1,
t: '$_id.t',
_id: 0,
},
},
],
});
} else {
return rawQuery(
`select
count(*) x count(*) x
${eventName ? `,event_name eventName` : ''} ${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''} ${urlPath ? `,url_path urlPath` : ''}
@ -72,8 +182,9 @@ async function relationalQuery(
${eventName ? `and eventName = $5` : ''} ${eventName ? `and eventName = $5` : ''}
${getEventDataFilterQuery(filters, params)} ${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`, ${timeSeries ? 'group by t' : ''}`,
params, params,
); );
}
} }
async function clickhouseQuery( async function clickhouseQuery(

95
queries/analytics/pageview/getPageviewMetrics.ts
View File

@ -31,7 +31,8 @@ async function relationalQuery(
}, },
) { ) {
const { startDate, endDate, filters = {}, column } = criteria; const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters, toUuid } = prisma; const { getDatabaseType, rawQuery, parseFilters, toUuid, client } = prisma;
const db = getDatabaseType();
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt); const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [ const params: any = [
@ -43,16 +44,99 @@ async function relationalQuery(
]; ];
let excludeDomain = ''; let excludeDomain = '';
let excludeDomainMongo = {};
if (column === 'referrer_domain') { if (column === 'referrer_domain') {
excludeDomain = 'and website_event.referrer_domain != $6'; excludeDomain = 'and website_event.referrer_domain != $6';
excludeDomainMongo = {
$ne: ['$referrer_domain', website.domain],
};
params.push(website.domain); params.push(website.domain);
} }
const { filterQuery, joinSession } = parseFilters(filters, params); const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery( if (db === 'mongodb') {
`select ${column} x, count(*) y return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', params[4]],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
excludeDomainMongo,
],
},
},
},
{
$group: {
_id: '$' + column,
y: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id',
y: 1,
_id: 0,
},
},
{
$sort: {
x: 1,
},
},
{
$sort: {
y: -1,
},
},
{
$limit: 100,
},
],
});
} else {
return rawQuery(
`select ${column} x, count(*) y
from website_event from website_event
${joinSession} ${joinSession}
where website_event.website_id = $1${toUuid()} where website_event.website_id = $1${toUuid()}
@ -64,8 +148,9 @@ async function relationalQuery(
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100`,
params, params,
); );
}
} }
async function clickhouseQuery( async function clickhouseQuery(

100
queries/analytics/pageview/getPageviewStats.ts
View File

@ -52,20 +52,112 @@ async function relationalQuery(
const params: any = [websiteId, resetDate, startDate, endDate]; const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params); const { filterQuery, joinSession } = parseFilters(filters, params);
//TODO: 구현해야 함 let sessionInclude = '';
let sessionGroupAggregation: any = { $match: {} };
let sessionProjectAggregation: any = { $match: {} };
if (count !== '*') {
sessionInclude = 'session_id : 1';
sessionGroupAggregation = {
$group: {
_id: {
t: '$t',
session_id: '$session_id',
},
},
};
sessionProjectAggregation = {
$project: {
t: '$_id.t',
},
};
}
if (db === 'mongodb') { if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({ return await client.websiteEvent.aggregateRaw({
pipeline: [ pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.pageView],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$project: {
t: {
$dateTrunc: {
date: '$created_at',
unit: unit,
},
},
sessionInclude,
},
},
sessionGroupAggregation,
sessionProjectAggregation,
{
$group: {
_id: {
t: '$t',
session_id: '$session_id',
},
y: {
$sum: 1,
},
},
},
{ {
$project: { $project: {
x: { x: {
$dateToString: { $dateToString: {
date: '$created_at', date: '$_id.t',
format: getDateQuery('', unit, timezone),
timezone: timezone, timezone: timezone,
format: getDateQuery('website_event.created_at', unit, timezone),
}, },
}, },
y: 1,
_id: 0,
},
},
{
$sort: {
x: 1,
}, },
}, },
], ],

103
queries/analytics/session/getSessionMetrics.ts
View File

@ -23,12 +23,103 @@ async function relationalQuery(
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt); const resetDate = new Date(website?.resetAt || website?.createdAt);
const { startDate, endDate, column, filters = {} } = criteria; const { startDate, endDate, column, filters = {} } = criteria;
const { toUuid, parseFilters, rawQuery } = prisma; const { getDatabaseType, toUuid, parseFilters, rawQuery, client } = prisma;
const db = getDatabaseType();
const params: any = [websiteId, resetDate, startDate, endDate]; const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params); const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery( if (db === 'mongodb') {
`select ${column} x, count(*) y return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$group: {
_id: '$session_id',
},
},
{
$lookup: {
from: 'session',
localField: '_id',
foreignField: '_id',
as: 'session',
},
},
{
$project: {
session: {
$arrayElemAt: ['$session', 0],
},
},
},
{
$group: {
_id: '$session.' + column,
sum: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id',
y: '$sum',
_id: 0,
},
},
{
$sort: {
sum: -1,
},
},
{
$limit: 100,
},
],
});
} else {
return rawQuery(
`select ${column} x, count(*) y
from session as x from session as x
where x.session_id in ( where x.session_id in (
select website_event.session_id select website_event.session_id
@ -44,10 +135,10 @@ async function relationalQuery(
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100`,
params, params,
); );
}
} }
async function clickhouseQuery( async function clickhouseQuery(
websiteId: string, websiteId: string,
data: { startDate: Date; endDate: Date; column: string; filters: object }, data: { startDate: Date; endDate: Date; column: string; filters: object },

44
queries/analytics/stats/getWebsiteStats.ts
View File

@ -39,6 +39,50 @@ async function relationalQuery(
if (db === 'mongodb') { if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({ return await client.websiteEvent.aggregateRaw({
pipeline: [ pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.pageView],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{ {
$project: { $project: {
session_id: '$session_id', session_id: '$session_id',