ambiser
/
umami
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: sql to mongo pipeline

pull/2040/head
minkik 2023-05-18 09:10:35 +09:00 committed by Joseph Lee
parent d5fac29dac
commit ed29fc08e8
7 changed files with 558 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pages/api/websites/[id]/stats.ts
View File

@ -92,14 +92,22 @@ export default async (
city,
},
});
const stats = Object.keys(metrics[0]).reduce((obj, key) => {
obj[key] = {
value: Number(metrics[0][key]) || 0,
change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0,
};
return obj;
}, {});
let stats: object = {
_id: '',
pageviews: 0,
uniques: 0,
bounces: 0,
totaltime: 0,
};
if (metrics.length != 0) {
stats = Object.keys(metrics[0]).reduce((obj, key) => {
obj[key] = {
value: Number(metrics[0][key]) || 0,
change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0,
};
return obj;
}, {});
}
return ok(res, stats);
}

104
queries/analytics/event/getEventMetrics.ts
View File

@ -45,14 +45,107 @@ async function relationalQuery(
};
},
) {
const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma;
const { getDatabaseType, toUuid, rawQuery, getDateQuery, getFilterQuery, client } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [websiteId, resetDate, startDate, endDate];
const filterQuery = getFilterQuery(filters, params);
const db = getDatabaseType();
return rawQuery(
`select
if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.customEvent],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$project: {
t: {
$dateTrunc: {
date: '$created_at',
unit: unit,
timezone: timezone,
},
},
event_name: 1,
},
},
{
$group: {
_id: {
t: '$t',
name: '$event_name',
},
y: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id.name',
t: {
$dateToString: {
date: '$_id.t',
format: getDateQuery('', unit, timezone),
timezone: timezone,
},
},
y: 1,
_id: 0,
},
},
{
$sort: {
t: 1,
},
},
],
});
} else {
return rawQuery(
`select
event_name x,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
@ -64,8 +157,9 @@ async function relationalQuery(
${filterQuery}
group by 1, 2
order by 2`,
params,
);
params,
);
}
}
async function clickhouseQuery(

121
queries/analytics/eventData/getEventData.ts
View File

@ -47,13 +47,123 @@ async function relationalQuery(
},
) {
const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
const { getDatabaseType, toUuid, rawQuery, getEventDataFilterQuery, getDateQuery, client } =
prisma;
const db = getDatabaseType();
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [websiteId, resetDate, startDate, endDate, eventName || ''];
return rawQuery(
`select
if (db === 'mongodb') {
let joinAggregation: any = { match: {} };
let matchAggregation: any = { match: {} };
let eventTypeProjectProperty = '';
let urlProjectProperty = '';
if (eventName || urlPath) {
joinAggregation = {
$lookup: {
from: 'website_event',
localField: 'website_event_id',
foreignField: '_id',
as: 'result',
},
};
eventTypeProjectProperty = 'event_name: {$arrayElemAt: ["$result.event_name", 0]}';
}
if (eventName) {
matchAggregation = {
$match: {
'result.event_name': eventName,
},
};
}
if (urlPath) {
urlProjectProperty = 'url_path: {$arrayElemAt: ["$result.url_path", 0],}';
}
let timeProjectProperty = '';
if (timeSeries) {
timeProjectProperty = `t: $dateTrunc: {date: "$created_at",unit: ${timeSeries.unit}, timezone : ${timeSeries.timezone}`;
}
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
joinAggregation,
matchAggregation,
{
$project: {
eventTypeProjectProperty,
timeProjectProperty,
urlProjectProperty,
},
},
{
$group: {
_id: {
url_path: '$url_path',
event_name: '$event_name',
t: '$t',
},
x: {
$sum: 1,
},
},
},
{
$project: {
url_path: '$_id.url_path',
urlPath: '$_id.url_path',
event_name: '$_id.event_name',
eventName: '$_id.event_name',
x: 1,
t: '$_id.t',
_id: 0,
},
},
],
});
} else {
return rawQuery(
`select
count(*) x
${eventName ? `,event_name eventName` : ''}
${urlPath ? `,url_path urlPath` : ''}
@ -72,8 +182,9 @@ async function relationalQuery(
${eventName ? `and eventName = $5` : ''}
${getEventDataFilterQuery(filters, params)}
${timeSeries ? 'group by t' : ''}`,
params,
);
params,
);
}
}
async function clickhouseQuery(

95
queries/analytics/pageview/getPageviewMetrics.ts
View File

@ -31,7 +31,8 @@ async function relationalQuery(
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters, toUuid } = prisma;
const { getDatabaseType, rawQuery, parseFilters, toUuid, client } = prisma;
const db = getDatabaseType();
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt);
const params: any = [
@ -43,16 +44,99 @@ async function relationalQuery(
];
let excludeDomain = '';
let excludeDomainMongo = {};
if (column === 'referrer_domain') {
excludeDomain = 'and website_event.referrer_domain != $6';
excludeDomainMongo = {
$ne: ['$referrer_domain', website.domain],
};
params.push(website.domain);
}
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery(
`select ${column} x, count(*) y
if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', params[4]],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
excludeDomainMongo,
],
},
},
},
{
$group: {
_id: '$' + column,
y: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id',
y: 1,
_id: 0,
},
},
{
$sort: {
x: 1,
},
},
{
$sort: {
y: -1,
},
},
{
$limit: 100,
},
],
});
} else {
return rawQuery(
`select ${column} x, count(*) y
from website_event
${joinSession}
where website_event.website_id = $1${toUuid()}
@ -64,8 +148,9 @@ async function relationalQuery(
group by 1
order by 2 desc
limit 100`,
params,
);
params,
);
}
}
async function clickhouseQuery(

100
queries/analytics/pageview/getPageviewStats.ts
View File

@ -52,20 +52,112 @@ async function relationalQuery(
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
//TODO: 구현해야 함
let sessionInclude = '';
let sessionGroupAggregation: any = { $match: {} };
let sessionProjectAggregation: any = { $match: {} };
if (count !== '*') {
sessionInclude = 'session_id : 1';
sessionGroupAggregation = {
$group: {
_id: {
t: '$t',
session_id: '$session_id',
},
},
};
sessionProjectAggregation = {
$project: {
t: '$_id.t',
},
};
}
if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.pageView],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$project: {
t: {
$dateTrunc: {
date: '$created_at',
unit: unit,
},
},
sessionInclude,
},
},
sessionGroupAggregation,
sessionProjectAggregation,
{
$group: {
_id: {
t: '$t',
session_id: '$session_id',
},
y: {
$sum: 1,
},
},
},
{
$project: {
x: {
$dateToString: {
date: '$created_at',
date: '$_id.t',
format: getDateQuery('', unit, timezone),
timezone: timezone,
format: getDateQuery('website_event.created_at', unit, timezone),
},
},
y: 1,
_id: 0,
},
},
{
$sort: {
x: 1,
},
},
],

103
queries/analytics/session/getSessionMetrics.ts
View File

@ -23,12 +23,103 @@ async function relationalQuery(
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || website?.createdAt);
const { startDate, endDate, column, filters = {} } = criteria;
const { toUuid, parseFilters, rawQuery } = prisma;
const { getDatabaseType, toUuid, parseFilters, rawQuery, client } = prisma;
const db = getDatabaseType();
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery(
`select ${column} x, count(*) y
if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$group: {
_id: '$session_id',
},
},
{
$lookup: {
from: 'session',
localField: '_id',
foreignField: '_id',
as: 'session',
},
},
{
$project: {
session: {
$arrayElemAt: ['$session', 0],
},
},
},
{
$group: {
_id: '$session.' + column,
sum: {
$sum: 1,
},
},
},
{
$project: {
x: '$_id',
y: '$sum',
_id: 0,
},
},
{
$sort: {
sum: -1,
},
},
{
$limit: 100,
},
],
});
} else {
return rawQuery(
`select ${column} x, count(*) y
from session as x
where x.session_id in (
select website_event.session_id
@ -44,10 +135,10 @@ async function relationalQuery(
group by 1
order by 2 desc
limit 100`,
params,
);
params,
);
}
}
async function clickhouseQuery(
websiteId: string,
data: { startDate: Date; endDate: Date; column: string; filters: object },

44
queries/analytics/stats/getWebsiteStats.ts
View File

@ -39,6 +39,50 @@ async function relationalQuery(
if (db === 'mongodb') {
return await client.websiteEvent.aggregateRaw({
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$event_type', EVENT_TYPE.pageView],
},
{
$eq: ['$website_id', websiteId],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: resetDate.toISOString(),
},
},
],
},
{
$gte: [
'$created_at',
{
$dateFromString: {
dateString: startDate.toISOString(),
},
},
],
},
{
lte: [
'$created_at',
{
$dateFromString: {
dateString: endDate.toISOString(),
},
},
],
},
],
},
},
},
{
$project: {
session_id: '$session_id',