diff --git a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts index 40a087410..6472521aa 100644 --- a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts +++ b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts @@ -397,8 +397,11 @@ function egressStatus(status: EgressDoc['status']): Record { return enumField('status', `EGRESS_${status.toUpperCase()}`); } -/** The request of an egress as the writer stores it: the set member of the oneof under its name. */ -function egressRequest(e: EgressDoc): Record { +/** + * The request of an egress as the writer stores it in `rawRequest` and in its events: the set member of the oneof under + * its name. + */ +export function egressRawRequest(e: EgressDoc): Record { const r = e.request as Partial< Record<'participantIdentity' | 'trackId' | 'videoTrackId' | 'audioTrackId', string> > & { @@ -452,7 +455,7 @@ function egressInfo(e: EgressDoc): Record { room_name: e.roomName, ...(e.sourceType === 'sdk' ? { source_type: 'EGRESS_SOURCE_TYPE_SDK' } : {}), started_at: unixNs(e.startedAt), - request: egressRequest(e) + request: egressRawRequest(e) }; } diff --git a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/generator.ts b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/generator.ts index bf7c435b6..013cee9b0 100644 --- a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/generator.ts +++ b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/generator.ts @@ -2,6 +2,7 @@ import { type Db, type Document, MongoBulkWriteError } from 'mongodb'; import { deriveEvents, deriveStats, + egressRawRequest, BASE_ROLLUPS, type DeriveWindows, type RollupOptions, @@ -253,7 +254,7 @@ export async function generate(db: Db, options: GenerateOptions): Promise; outputs: EgressOutput[]; /** Status transitions in order; at most 50 entries. */ statusHistory: EgressStatusChange[]; diff --git a/openvidu-dashboard/openspec/changes/egress-raw-request/.openspec.yaml b/openvidu-dashboard/openspec/changes/egress-raw-request/.openspec.yaml new file mode 100644 index 000000000..97e364202 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/egress-raw-request/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-27 diff --git a/openvidu-dashboard/openspec/changes/egress-raw-request/design.md b/openvidu-dashboard/openspec/changes/egress-raw-request/design.md new file mode 100644 index 000000000..c6b944680 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/egress-raw-request/design.md @@ -0,0 +1,55 @@ +# Design + +## Context + +- The codec encodes a protocol message through the allow-list: protocol field names, enum values as names, the set + member of a oneof as a document named after the oneof, URLs redacted by rule. Raw events store `EgressInfo` this + way, so `payload.egress.request` holds the full request without secrets. +- The egress projection writes `request` from `egressRequest()`, a summary built field by field. +- Validators are applied at every start with `validationLevel: strict` and the configured action (`warn` by + default). + +## Goals / Non-Goals + +**Goals:** +- The egress document keeps the full request, without secrets, for the entities retention. + +**Non-Goals:** +- Replacing the summary, which the list and the detail facts read. +- Backfilling stored documents. + +## Decisions + +### Same encoding as the raw payloads + +`rawRequest` is the `request` member of `EgressInfo` as the codec encodes it: one key, the oneof member that is set +(`room_composite`, `web`, `participant`, `track_composite`, `track`, `egress` or `replay`), holding the request. It is +the value `payload.egress.request` of the raw events of the same egress. The allow-list stays the only place that +decides what is stored. + +Alternative rejected: unwrapping the oneof member. The member tells the legacy room composite API apart from the +unified `StartEgress` and `ExportReplay` APIs, which `type` does not. + +Alternative rejected: protojson with camelCase names. Raw payloads and this field would show the same request in two +namings. + +### Written on every event + +Every `EGRESS_*` event carries the request, so the field is set on each upsert, like `request`. It is absent when the +event has no request. + +### Data model + +- Collection `egresses`: new optional field `rawRequest` (object). No index; the field is never queried. +- Retention: the entities retention of the document. +- Schema version: unchanged (1). The field is optional, and the contract has not shipped in a release. A dashboard + that does not know the field ignores it. +- Migration from legacy events: the legacy migration decodes the legacy `EgressInfo` and projects it like a live + event, so a migrated egress gets `rawRequest` when its legacy events carry a request. +- Performance: one encode of a message the codec already encodes for the raw event. The document grows by the size of + the request, usually under 1 KB. + +## Risks / Trade-offs + +- [A future allow-list entry stores a secret in the document too] → The codec tests keep checking every allow-listed + field against the secret patterns, and an edition that shows the field can filter it again on read. diff --git a/openvidu-dashboard/openspec/changes/egress-raw-request/proposal.md b/openvidu-dashboard/openspec/changes/egress-raw-request/proposal.md new file mode 100644 index 000000000..22f87c5e9 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/egress-raw-request/proposal.md @@ -0,0 +1,43 @@ +# Proposal + +## Why + +The `request` of an egress document is a summary: source, encoding and the kinds of output. The configuration of each +output (file type and path, stream protocol and URLs, segment and image settings), the audio routes of the unified API +and the replay offsets are lost. The full request, through the secrets allow-list, exists only in the raw `EGRESS_*` +event payloads, which expire with the events retention while the egress document keeps the entities retention. + +## What Changes + +- The `egresses` document gains an optional `rawRequest`: the request of the egress encoded as raw event payloads + encode `egress.request`, with the protocol field names, the set member of each oneof under the oneof name and only + the allow-listed fields. Credentials, storage uploads and webhooks are never stored, and URLs are redacted. +- `request` keeps its summary, unchanged. + +Scope excludes the following: +- Egress documents already stored, which have no `rawRequest` until their egress receives another event. +- Any change to the allow-list or to the request summary. +- How an edition shows the field. + +Edition: CE and PRO (data model). Repositories: dashboard (contract, typings, data generator, specs) and +openvidu-livekit (projection and document types). + +CHANGELOG: none. The analytics data model has not shipped in a release. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `data-model`: the egress document stores the request in raw payload encoding. + +## Impact + +- Contract (`dashboard-ce/typings`): `EgressDocument.rawRequest`, the `egresses` validator and the egress fixtures; the + contract version stays 1. +- Writer: the egress projection, the `EgressDocument` struct, the vendored contract and the integration tests. +- Data generator: generated egress documents carry `rawRequest`. +- Dashboard backend: none; the egress list projection does not read the field. diff --git a/openvidu-dashboard/openspec/changes/egress-raw-request/specs/data-model/spec.md b/openvidu-dashboard/openspec/changes/egress-raw-request/specs/data-model/spec.md new file mode 100644 index 000000000..640f5f23a --- /dev/null +++ b/openvidu-dashboard/openspec/changes/egress-raw-request/specs/data-model/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: Egress document +Edition: CE and PRO. An `egresses` document SHALL describe one egress: room reference, type, source, +lifecycle status with history, timestamps, error, a sanitized request summary, the request in raw payload encoding and +its outputs, with the field list of the shared contract. `roomNameLower` SHALL be the lower-cased `roomName`, written +together with it. `rawRequest` SHALL hold the request as the raw events of the egress store `egress.request`: the +member of the request oneof that is set, under its protocol name, through the secrets allow-list. + +#### Scenario: Fields of an egress document +- **WHEN** an egress document is read +- **THEN** it has `roomSid`, `roomName`, `roomNameLower`, `type` (`room_composite` | `web` | `participant` | `track_composite` | `track`), `sourceType`, `status` (`starting` | `active` | `ending` | `complete` | `failed` | `aborted` | `limit_reached`), `closedBy`, `startedAt`, `updatedAt`, `endedAt`, `durationMs`, `error`, `errorCode`, `request` (sanitized), `rawRequest?`, `outputs[] { kind, name or url, location, size, durationMs, status, error, startedAt, endedAt }`, `statusHistory[]` with at most 50 entries, `retryCount`, `backupStorageUsed`, `schemaVersion`, `expiresAt` + +#### Scenario: Egress completes +- **GIVEN** `EGRESS_STARTED`, two `EGRESS_UPDATED` and `EGRESS_ENDED` with status COMPLETE and one file result +- **WHEN** all are processed +- **THEN** the document has `status: "complete"`, one output of kind `file` with its filename, location, size and duration, and four `statusHistory` entries + +#### Scenario: Credentials are never stored +- **GIVEN** an egress request with S3 credentials and a stream URL containing a key +- **WHEN** the egress is projected +- **THEN** neither `request` nor `rawRequest` contains a credential field, and the stream output URL has its key redacted + +#### Scenario: Raw request +- **GIVEN** a room composite egress of room `Demo-Room` with layout `grid`, preset `H264_720P_30`, an MP4 file output with path `recordings/demo.mp4` uploaded to S3 with credentials, and a stream output to `rtmps://live.example.com/app/live_123?token=abc` +- **WHEN** its `EGRESS_STARTED` event is processed +- **THEN** `rawRequest` is `{ room_composite: { room_name: "Demo-Room", layout: "grid", options: { preset: "H264_720P_30" }, file_outputs: [{ file_type: "MP4", filepath: "recordings/demo.mp4" }], stream_outputs: [{ urls: ["rtmps://live.example.com/app/***"] }] } }`, the value of `payload.egress.request` in the stored event + +#### Scenario: Room name search field +- **GIVEN** an egress of room `Demo-Room` +- **WHEN** its document is read +- **THEN** it has `roomName: "Demo-Room"` and `roomNameLower: "demo-room"` + +#### Scenario: Web egress without room +- **GIVEN** a web egress, which has no room +- **WHEN** its document is read +- **THEN** it has neither `roomName` nor `roomNameLower` diff --git a/openvidu-dashboard/openspec/changes/egress-raw-request/tasks.md b/openvidu-dashboard/openspec/changes/egress-raw-request/tasks.md new file mode 100644 index 000000000..bc5cacfa3 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/egress-raw-request/tasks.md @@ -0,0 +1,31 @@ +# Tasks + +## 1. Typings + +- [x] 1.1 Add `rawRequest?: Record` to `EgressDocument` in `typings/src/data-model/entities.ts` and the + `rawRequest` object to the `egresses` schema in `typings/contract/schemas.ts`. Add the raw request of each egress + fixture in `contract/fixtures/egress-ingress.ts`, shared with the recording event of `contract/fixtures/events.ts`. + Regenerate the contract with `pnpm --filter @openvidu-dashboard/typings run contract`. Verify with the typings + typecheck and tests. + +## 2. Writer (openvidu-livekit) + +- [x] 2.1 Vendor the contract with `typings/contract/vendor.sh`. Add `RawRequest` to the `EgressDocument` struct and set + `rawRequest` in the egress projection from the `request` member of the encoded `EgressInfo`. Verify with the unit + tests of `openvidu/analytics/...` and the lint of the fork. +- [x] 2.2 Assert in the egress integration tests that `rawRequest` holds the file and stream outputs, with the stream + URL redacted and no credential. Verify with the analytics integration tests and `typings/contract/writer-check.sh`. + +## 3. Data generator + +- [x] 3.1 Write `rawRequest` on every generated egress document, from the request the generator derives for the egress + events. Verify with the data generator tests and by loading a demo dataset. + +## 4. Docs and CHANGELOG + +- [x] 4.1 No CHANGELOG line: the analytics data model has not shipped. Update the `AGENTS.md` files only if a command or + path changes. Verify by reading the change against them. + +## 5. Integration check + +- [x] 5.1 Run `./dashboard.sh check` and the backend integration tests. Verify that they pass.