import { writeFileSync } from 'node:fs'; import { generate } from '@openvidu-dashboard/data-generator/generator'; import { queryContext, type QueryContext } from '@openvidu-dashboard/data-generator/context'; import { type Db, type Document, MongoClient } from 'mongodb'; import { afterAll, beforeAll, describe, expect, inject, it } from 'vitest'; import { Collection, ParticipantKind, ParticipantStatus, type RoomDocument, RoomStatus } from '@openvidu-dashboard/typings'; import { decodeCursor } from '../../src/api/cursor.js'; import { rankedPipeline } from '../../src/api/ranked.js'; import { prefixRegex } from '../../src/api/search.js'; import { ROOM_LIST_PROJECTION, roomsListPlan } from '../../src/rooms/rooms-query.js'; import { STANDARD_SESSIONS } from '../../src/rooms/rooms-repository.js'; import { roomsListQuery } from '../../src/rooms/rooms-schemas.js'; import { RoomsService } from '../../src/rooms/rooms-service.js'; import { WriterFreshness } from '../../src/status/freshness.js'; import { expectWithinBudgets } from '../support/latency-budget.js'; import { expectIdLookup, expectIndexBacked, expectIndexes, p95 } from '../support/query-plan.js'; /** * Every rooms query explained on a database generated by the data generator, plus the p95 of * the performance budget. ROOMS_PLAN_SCENARIO (S by default) and ROOMS_PLAN_DAYS (2 by default) set the data; the * design budget is measured at scenario M for 7 days. ROOMS_PLAN_REPORT names a file that also receives the report. */ const mongoUri = inject('mongoUri'); const scenario = (process.env.ROOMS_PLAN_SCENARIO ?? 'S') as 'S' | 'M' | 'L'; const days = Number(process.env.ROOMS_PLAN_DAYS ?? '2'); const DATABASE = 'dashboard_test_rooms_plan'; const LIMIT = 100; const DAY = 86_400_000; /** Generating scenario M or L for several days takes longer than the project's hook timeout. */ const GENERATION_TIMEOUT_MS = scenario === 'S' ? 600_000 : 3_600_000; describe.skipIf(mongoUri === '')(`rooms query plan at scenario ${scenario}`, () => { let client: MongoClient; let db: Db; let ctx: QueryContext; let service: RoomsService; const now = new Date(); const report: string[] = []; const listParams = (query: Record) => roomsListQuery(() => now).parse({ from: ctx.range.from.toISOString(), limit: String(LIMIT), ...query }); const explainList = async (query: Record, bound: (branches: number) => number) => { const params = listParams(query); const plan = roomsListPlan(params); if (plan.kind === 'ranked') { const inRange = await db.collection(Collection.ROOMS).countDocuments(plan.countFilter); const request = { ...params, match: plan.match, rank: plan.rank, projection: ROOM_LIST_PROJECTION }; return expectIndexBacked( db, { collection: Collection.ROOMS, pipeline: rankedPipeline({ ...request, cursor: undefined, now }, now) }, { maxDocsExamined: inRange } ); } const branches = Array.isArray(plan.filter.$or) ? plan.filter.$or.length : 1; return expectIndexBacked( db, db .collection(Collection.ROOMS) .find(plan.filter, { projection: ROOM_LIST_PROJECTION }) .sort(plan.kind === 'page' ? plan.sort : { _id: 1 }) .limit(LIMIT + 1), { maxDocsExamined: bound(branches) } ); }; beforeAll(async () => { client = new MongoClient(mongoUri); db = client.db(DATABASE); await db.dropDatabase(); await generate(db, { scenario, days, statsMode: 'participant', seed: 1, retentionDays: { events: 7, stats: 7, tracks: 7, entities: 90 }, timeseries: true }); ctx = await queryContext(db); service = new RoomsService(db, { totalMaxRangeMs: 7 * DAY, freshness: new WriterFreshness(db, () => now), now: () => now }); }, GENERATION_TIMEOUT_MS); afterAll(async () => { if (report.length > 0) { const text = [`Rooms p95 of 20 runs, scenario ${scenario}, ${String(days)} days`, ...report].join('\n'); console.log(text); if (process.env.ROOMS_PLAN_REPORT !== undefined) { writeFileSync(process.env.ROOMS_PLAN_REPORT, `${text}\n`); } } await db.dropDatabase(); await client.close(); }); describe('list', () => { const page = () => LIMIT + 2; it('status set, sort createdAt: status_createdAt_id', async () => { expectIndexes(await explainList({ status: 'closed' }, page), ['status_createdAt_id']); }); it('status all, sort createdAt: createdAt_id', async () => { expectIndexes(await explainList({}, page), ['createdAt_id']); }); it.each(['endedAt', 'duration', 'uniqueIdentities', 'active'])( 'ranked by %s: the range index of the filters, at most the rooms in range', async (sort) => { const { nodeId } = (await db.collection(Collection.ROOMS).findOne({})) ?? {}; expectIndexes(await explainList({ sort }, page), ['createdAt_id']); expectIndexes(await explainList({ sort, status: 'closed' }, page), ['status_createdAt_id']); expectIndexes(await explainList({ sort, nodeId: nodeId ?? 'none' }, page), [ 'nodeId_status_createdAt_id' ]); expectIndexes(await explainList({ sort, features: 'egress' }, page), ['status_createdAt_id_egress']); expectIndexes(await explainList({ sort, q: ctx.samples.namePrefix }, page), ['nameLower_createdAt']); } ); it('name prefix within one node: nameLower_createdAt, node residual', async () => { const { nodeId } = (await db.collection(Collection.ROOMS).findOne({})) ?? {}; const inPrefix = await db .collection(Collection.ROOMS) .countDocuments({ nameLower: prefixRegex(ctx.samples.namePrefix) }); expectIndexes(await explainList({ q: ctx.samples.namePrefix, nodeId: nodeId ?? 'none' }, () => inPrefix), [ 'nameLower_createdAt' ]); }); it('features: one partial index per flag', async () => { expectIndexes(await explainList({ features: 'egress' }, page), ['status_createdAt_id_egress']); expectIndexes(await explainList({ features: 'egress,agent' }, (branches) => branches * (LIMIT + 2)), [ 'status_createdAt_id_egress', 'status_createdAt_id_agent' ]); expectIndexes( await explainList({ features: 'ingress,sip', status: 'closed' }, (branches) => branches * (LIMIT + 2)), ['status_createdAt_id_ingress', 'status_createdAt_id_sip'] ); }); it('name prefix: nameLower_createdAt within the prefix range', async () => { const inPrefix = await db .collection(Collection.ROOMS) .countDocuments({ nameLower: prefixRegex(ctx.samples.namePrefix) }); expectIndexes(await explainList({ q: ctx.samples.namePrefix }, () => inPrefix), ['nameLower_createdAt']); }); it('SID: _id', async () => { expectIdLookup(await explainList({ q: ctx.samples.roomSid }, () => 1)); }); it('node: nodeId_status_createdAt_id', async () => { const { nodeId } = (await db.collection(Collection.ROOMS).findOne({})) ?? {}; expectIndexes(await explainList({ nodeId: nodeId ?? 'none' }, page), ['nodeId_status_createdAt_id']); }); it('a page after a cursor keeps the index and examines the page and the ties', async () => { const first = await service.list(listParams({ status: 'closed', limit: '25' })); const cursor = decodeCursor(first.nextCursor ?? '', 'createdAt', 'desc'); if (typeof cursor === 'string') { throw new Error(cursor); } const plan = roomsListPlan(listParams({ status: 'closed', limit: '25' }), cursor); if (plan.kind !== 'page') { throw new Error('createdAt is index-ordered'); } expectIndexes( await expectIndexBacked( db, db.collection(Collection.ROOMS).find(plan.filter).sort(plan.sort).limit(26), { maxDocsExamined: 27 } ), ['status_createdAt_id'] ); }); it.each([ ['closed rooms', { status: 'closed' }], ['two features', { features: 'egress,agent' }] ])('walks forward and back on %s without gaps or duplicates', async (_name, query) => { const params = listParams({ ...query, limit: '10' }); const plan = roomsListPlan(params); const truth = ( await db .collection(Collection.ROOMS) .find(plan.kind === 'page' ? plan.countFilter : {}, { projection: { _id: 1 } }) .sort({ createdAt: -1, _id: -1 }) .limit(30) .toArray() ).map((room) => room._id); expect(truth.length).toBeGreaterThan(20); const first = await service.list(params); const second = await service.list({ ...params, cursor: first.nextCursor ?? '' }); const third = await service.list({ ...params, cursor: second.nextCursor ?? '' }); const back = await service.list({ ...params, cursor: third.prevCursor ?? '' }); const ids = (p: { items: { sid: string }[] }) => p.items.map((item) => item.sid); expect([...ids(first), ...ids(second), ...ids(third)].slice(0, truth.length)).toEqual(truth); expect(ids(back)).toEqual(ids(second)); }); }); describe('summary', () => { const matched = (collection: string, match: Document) => async () => db.collection(collection).countDocuments(match); it('summary counts', async () => { const range = { $gte: new Date(now.getTime() - DAY), $lt: now }; // Any index with the right prefix serves a count; the planner may prefer the smaller partial one, which reads // every active session and checks the kind after the fetch, so the documents it may examine bound the count. const counts: [string, Document, string[], Document?][] = [ [ Collection.ROOMS, { status: RoomStatus.ACTIVE }, ['status_createdAt_id', 'status_endedAt_id', 'status_lastEventAt_active'] ], [ Collection.PARTICIPANTS, { status: ParticipantStatus.ACTIVE, kind: ParticipantKind.STANDARD }, ['status_kind', 'status_lastStatsAt_open', 'kind_status_joinedAt_id'], { status: ParticipantStatus.ACTIVE } ], [Collection.ROOMS, { createdAt: range }, ['createdAt_id']], [Collection.PARTICIPANTS, { ...STANDARD_SESSIONS, joinedAt: range }, ['kind_status_joinedAt_id']] ]; for (const [collection, match, allowed, examined = match] of counts) { const plan = await expectIndexBacked( db, { collection, pipeline: [{ $match: match }, { $count: 'n' }] }, { maxDocsExamined: await matched(collection, examined)() } ); expect(plan.collscan).toBe(false); expect(plan.indexes.length).toBeGreaterThan(0); expect(plan.indexes.every((index) => allowed.includes(index))).toBe(true); } expectIndexes( await expectIndexBacked( db, { collection: Collection.ANALYTICS_HOURLY, pipeline: [ { $match: { _id: { $gte: 'global:', $lte: 'global:~' } } }, { $group: { _id: null, rooms: { $sum: '$rooms.created' } } } ] }, { maxDocsExamined: await matched(Collection.ANALYTICS_HOURLY, { scope: 'global' })() } ), ['_id_'] ); }); }); describe('performance budget', () => { it('reports the p95 of every budgeted query', async () => { const biggest = await db.collection(Collection.ROOMS).findOne({ _id: ctx.samples.roomSid }); const common = (await db.collection(Collection.ROOMS).findOne({}))?.nameLower.slice(0, 2) ?? 'ro'; const rows: [string, string, () => Promise][] = [ ['List, status all (100 rows)', '150 ms', () => service.list(listParams({}))], ...(['endedAt', 'duration', 'uniqueIdentities', 'active'] as const).map( (sort): [string, string, () => Promise] => [ `List, ranked by ${sort}`, '300 ms', () => service.list(listParams({ sort })) ] ), ['List, two features', '150 ms', () => service.list(listParams({ features: 'egress,agent' }))], ['List, node', '150 ms', () => service.list(listParams({ nodeId: biggest?.nodeId ?? 'none' }))], [`List, prefix "${common}"`, '150 ms', () => service.list(listParams({ q: common }))], [ 'Summary, exact (24 h)', '150 ms', () => service.summary({ from: new Date(now.getTime() - DAY), to: now }) ], [ 'Summary, rollups (30 d)', '100 ms', () => service.summary({ from: new Date(now.getTime() - 30 * DAY), to: now }) ] ]; report.push('| Query | Budget | p95 |', '| --- | --- | --- |'); const overruns: string[] = []; for (const [name, budget, run] of rows) { const measured = await p95(run); report.push(`| ${name} | ${budget} | ${measured.toFixed(1)} ms |`); if (name.startsWith('List, ranked') && measured > Number.parseFloat(budget)) { overruns.push(`${name}: ${measured.toFixed(1)} ms > ${budget}`); } } expect(report.length).toBe(rows.length + 2); expectWithinBudgets(overruns); }); }); });