import { writeFileSync } from 'node:fs'; import { generate } from '@openvidu-dashboard/data-generator/generator'; import { queryContext, type QueryContext } from '@openvidu-dashboard/data-generator/context'; import { type Db, MongoClient } from 'mongodb'; import { afterAll, beforeAll, describe, expect, inject, it } from 'vitest'; import { Collection, EGRESS_IN_FLIGHT_STATUSES, type EgressDocument } from '@openvidu-dashboard/typings'; import { pageQuery } from '../../src/api/cursor.js'; import { rankedPipeline } from '../../src/api/ranked.js'; import { prefixRegex } from '../../src/api/search.js'; import { EGRESS_LIST_PROJECTION } from '../../src/egress/egress-mappers.js'; import { egressesListPlan } from '../../src/egress/egress-query.js'; import { egressesListQuery } from '../../src/egress/egress-schemas.js'; import { EgressService } from '../../src/egress/egress-service.js'; import { expectIdLookup, expectIndexBacked, expectIndexes, p95 } from '../support/query-plan.js'; import { expectWithinBudgets } from '../support/latency-budget.js'; /** * Every egress query explained, filter by filter, on a database generated by the data generator, plus * the p95 of the performance budget, which fails the suite when exceeded. EGRESS_PLAN_SCENARIO (S by default), * EGRESS_PLAN_DAYS (2 by default) and EGRESS_PLAN_REPORT (a file that also receives the report) work as in the rooms * suite. */ const mongoUri = inject('mongoUri'); const scenario = (process.env.EGRESS_PLAN_SCENARIO ?? 'S') as 'S' | 'M' | 'L'; const days = Number(process.env.EGRESS_PLAN_DAYS ?? '2'); const DATABASE = 'dashboard_test_egress_plan'; const LIMIT = 100; const DAY = 86_400_000; const GENERATION_TIMEOUT_MS = scenario === 'S' ? 600_000 : 3_600_000; describe.skipIf(mongoUri === '')(`egress query plan at scenario ${scenario}`, () => { let client: MongoClient; let db: Db; let ctx: QueryContext; let service: EgressService; let sample: EgressDocument; /** An egress of a room and a prefix of its room name, so that the prefix always matches it. */ let roomNameSample: { roomSid: string; prefix: string }; const now = new Date(); const report: string[] = []; const listParams = (query: Record) => egressesListQuery(() => now).parse({ from: ctx.range.from.toISOString(), limit: String(LIMIT), ...query }); const egresses = () => db.collection(Collection.EGRESSES); const raw = () => db.collection(Collection.EGRESSES); /** The first page of a list request as the service reads it on the index of its plan. */ const explainList = async (query: Record, bound: number) => { const params = listParams(query); const plan = egressesListPlan(params); if (plan.kind === 'id') { throw new Error('expected a page or ranked plan'); } const range = { from: params.from, to: params.to }; const page = pageQuery('startedAt', params.order, undefined, range); if (plan.kind === 'ranked') { const match = { ...plan.filter, ...page.filter }; const inRange = await raw().countDocuments(match); const request = { ...params, match, rank: plan.rank, projection: EGRESS_LIST_PROJECTION, cursor: undefined, now }; return expectIndexBacked( db, { collection: Collection.EGRESSES, pipeline: rankedPipeline(request, now), options: { hint: plan.hint } }, { maxDocsExamined: inRange } ); } const direction: 1 | -1 = params.order === 'asc' ? 1 : -1; const sorted = plan.hint === 'startedAt_id' || plan.hint === 'roomNameLower_startedAt'; return expectIndexBacked( db, raw() .find({ ...plan.filter, ...page.filter }, { projection: EGRESS_LIST_PROJECTION, hint: plan.hint }) .sort(sorted ? page.sort : { startedAt: direction }) .limit(LIMIT + 1), { maxDocsExamined: bound } ); }; beforeAll(async () => { client = new MongoClient(mongoUri); db = client.db(DATABASE); await db.dropDatabase(); await generate(db, { scenario, days, statsMode: 'participant', seed: 1, retentionDays: { events: 7, stats: 7, tracks: 7, entities: 90 }, timeseries: true }); ctx = await queryContext(db); service = new EgressService(db, { totalMaxRangeMs: 7 * DAY, now: () => now }); const found = await egresses().findOne({ _id: ctx.samples.egressId }); if (found === null) { throw new Error('the generated database has no sample egress'); } sample = found; const ofRoom = await egresses().findOne({ roomNameLower: { $exists: true } }); if (ofRoom?.roomSid === undefined || ofRoom.roomNameLower === undefined) { throw new Error('the generated database has no egress of a room'); } roomNameSample = { roomSid: ofRoom.roomSid, prefix: ofRoom.roomNameLower.slice(0, -1) }; }, GENERATION_TIMEOUT_MS); afterAll(async () => { if (report.length > 0) { const text = [`Egress p95 of 20 runs, scenario ${scenario}, ${String(days)} days`, ...report].join('\n'); console.log(text); if (process.env.EGRESS_PLAN_REPORT !== undefined) { writeFileSync(process.env.EGRESS_PLAN_REPORT, `${text}\n`); } } await db.dropDatabase(); await client.close(); }); it('status=all: startedAt_id without a sort, type residual', async () => { const inRange = await egresses().countDocuments({ startedAt: { $gte: ctx.range.from, $lt: now } }); const all = await explainList({}, LIMIT + 1); expectIndexes(all, ['startedAt_id']); expect(all.planStages).not.toContain('SORT'); expectIndexes(await explainList({ type: 'web' }, inRange), ['startedAt_id']); }); it('status set: status_startedAt merged in startedAt order', async () => { const plan = await explainList({ status: EGRESS_IN_FLIGHT_STATUSES.join(',') }, LIMIT + 1); expectIndexes(plan, ['status_startedAt']); expect(plan.planStages).not.toContain('SORT'); expectIndexes(await explainList({ status: 'failed,aborted', type: 'track' }, LIMIT * 60), ['status_startedAt']); }); it('room: roomSid_startedAt, status and type residual', async () => { const ofRoom = await raw().countDocuments({ roomSid: sample.roomSid }); expectIndexes(await explainList({ room: sample.roomSid ?? 'RM_none', status: 'complete' }, ofRoom), [ 'roomSid_startedAt' ]); }); it('room name prefix: roomNameLower_startedAt within the prefix, no room cap', async () => { const { roomSid, prefix } = roomNameSample; const inPrefix = await raw().countDocuments({ roomNameLower: prefixRegex(prefix) }); const ofRoom = await raw().countDocuments({ roomSid }); expect(inPrefix).toBeGreaterThan(0); expectIndexes(await explainList({ q: prefix }, inPrefix), ['roomNameLower_startedAt']); // With a room filter the room index serves the query and the prefix is a residual filter. expectIndexes(await explainList({ q: prefix, room: roomSid }, ofRoom), ['roomSid_startedAt']); }); it.each(['endedAt', 'duration'])( 'ranked by %s: the range index of the filters, at most the egresses in range', async (sort) => { expectIndexes(await explainList({ sort }, LIMIT), ['startedAt_id']); expectIndexes(await explainList({ sort, status: 'complete' }, LIMIT), ['status_startedAt']); expectIndexes(await explainList({ sort, room: sample.roomSid ?? 'RM_none' }, LIMIT), ['roomSid_startedAt']); expectIndexes(await explainList({ sort, q: roomNameSample.prefix }, LIMIT), ['roomNameLower_startedAt']); } ); it('id, detail and summary: _id, status_startedAt and startedAt_id', async () => { expectIdLookup( await expectIndexBacked(db, egresses().find({ _id: sample._id }).limit(1), { maxDocsExamined: 1 }) ); const plan = egressesListPlan(listParams({ q: sample._id })); expect(plan).toEqual({ kind: 'id', filter: { _id: sample._id } }); const active = await expectIndexBacked( db, egresses().find({ status: { $in: [...EGRESS_IN_FLIGHT_STATUSES] } }, { projection: { _id: 1 } }), { maxDocsExamined: () => Number.MAX_SAFE_INTEGER } ); expectIndexes(active, ['status_startedAt']); const figures = await expectIndexBacked( db, { collection: Collection.EGRESSES, pipeline: [ { $match: { startedAt: { $gte: new Date(now.getTime() - DAY), $lt: now } } }, { $count: 'n' } ] }, { maxDocsExamined: () => Number.MAX_SAFE_INTEGER, collscanThreshold: 0 } ); expect(figures.collscan).toBe(false); }); it('reports the p95 of every budgeted query and fails above its budget', async () => { const prefix = (sample.roomName ?? 'room').slice(0, 3).toLowerCase(); const rows: [string, number, () => Promise][] = [ ['List, status all (100 rows)', 150, () => service.list(listParams({}))], ['List, in flight', 150, () => service.list(listParams({ status: EGRESS_IN_FLIGHT_STATUSES.join(',') }))], ['List, failed', 150, () => service.list(listParams({ status: 'failed' }))], ['List, type=web', 300, () => service.list(listParams({ type: 'web' }))], ['List, room', 150, () => service.list(listParams({ room: sample.roomSid ?? 'RM_none' }))], [`List, room name prefix "${prefix}"`, 200, () => service.list(listParams({ q: prefix }))], ['List, ranked by endedAt', 200, () => service.list(listParams({ sort: 'endedAt' }))], ['List, ranked by duration', 200, () => service.list(listParams({ sort: 'duration' }))], ['Summary, exact (24 h)', 150, () => service.summary({ from: new Date(now.getTime() - DAY), to: now })], [ 'Summary, rollups (30 d)', 100, () => service.summary({ from: new Date(now.getTime() - 30 * DAY), to: now }) ] ]; report.push('| Query | Budget | p95 |', '| --- | --- | --- |'); const over: string[] = []; for (const [name, budget, run] of rows) { const measured = await p95(run); report.push(`| ${name} | ${String(budget)} ms | ${measured.toFixed(1)} ms |`); if (measured > budget) { over.push(`${name}: ${measured.toFixed(1)} ms > ${String(budget)} ms`); } } expectWithinBudgets(over); }); });