mirror of https://github.com/OpenVidu/openvidu.git
openvidu-dashboard: egress documents keep the request as sent
rawRequest holds the request in raw event payload encoding, through the writer allow-list, for the entities retention. The data generator writes it too.chore/update-openvidu-actions-1.0.25
juancarmore
parent
4be6333d02
commit
88debe2b46
|
|
@ -397,8 +397,11 @@ function egressStatus(status: EgressDoc['status']): Record<string, string> {
|
|||
return enumField('status', `EGRESS_${status.toUpperCase()}`);
|
||||
}
|
||||
|
||||
/** The request of an egress as the writer stores it: the set member of the oneof under its name. */
|
||||
function egressRequest(e: EgressDoc): Record<string, unknown> {
|
||||
/**
|
||||
* The request of an egress as the writer stores it in `rawRequest` and in its events: the set member of the oneof under
|
||||
* its name.
|
||||
*/
|
||||
export function egressRawRequest(e: EgressDoc): Record<string, unknown> {
|
||||
const r = e.request as Partial<
|
||||
Record<'participantIdentity' | 'trackId' | 'videoTrackId' | 'audioTrackId', string>
|
||||
> & {
|
||||
|
|
@ -452,7 +455,7 @@ function egressInfo(e: EgressDoc): Record<string, unknown> {
|
|||
room_name: e.roomName,
|
||||
...(e.sourceType === 'sdk' ? { source_type: 'EGRESS_SOURCE_TYPE_SDK' } : {}),
|
||||
started_at: unixNs(e.startedAt),
|
||||
request: egressRequest(e)
|
||||
request: egressRawRequest(e)
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { type Db, type Document, MongoBulkWriteError } from 'mongodb';
|
|||
import {
|
||||
deriveEvents,
|
||||
deriveStats,
|
||||
egressRawRequest,
|
||||
BASE_ROLLUPS,
|
||||
type DeriveWindows,
|
||||
type RollupOptions,
|
||||
|
|
@ -253,7 +254,7 @@ export async function generate(db: Db, options: GenerateOptions): Promise<Genera
|
|||
}
|
||||
|
||||
for (const doc of bundle.egresses) {
|
||||
writer.add(COLLECTIONS.egresses, doc);
|
||||
writer.add(COLLECTIONS.egresses, { ...doc, rawRequest: egressRawRequest(doc) });
|
||||
}
|
||||
|
||||
for (const doc of bundle.ingresses) {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,16 @@ import {
|
|||
schemaVersion
|
||||
} from './scenario.js';
|
||||
|
||||
/** The request of the recording, as its document and its raw events store it. */
|
||||
export const recordingRequest = {
|
||||
room_composite: {
|
||||
room_name: CLOSED_ROOM.name,
|
||||
layout: 'speaker',
|
||||
options: { preset: 'H264_720P_30' },
|
||||
file_outputs: [{ file_type: 'MP4', filepath: 'recordings/daily-standup-2026-10-12.mp4' }]
|
||||
}
|
||||
};
|
||||
|
||||
const recordingStart = at('09:02:00');
|
||||
const recordingEnd = at('09:40:20');
|
||||
const failedStart = at('09:30:00');
|
||||
|
|
@ -33,6 +43,7 @@ export const egresses: EgressDocument[] = [
|
|||
endedAt: recordingEnd,
|
||||
durationMs: between(recordingStart, recordingEnd),
|
||||
request: { layout: 'speaker', preset: 'H264_720P_30', outputs: ['file'] },
|
||||
rawRequest: recordingRequest,
|
||||
outputs: [
|
||||
{
|
||||
kind: 'file',
|
||||
|
|
@ -71,6 +82,13 @@ export const egresses: EgressDocument[] = [
|
|||
error: 'upload failed: access denied',
|
||||
errorCode: 403,
|
||||
request: { trackId: 'TR_aLiceMic0pus', outputs: ['file'] },
|
||||
rawRequest: {
|
||||
track: {
|
||||
room_name: CLOSED_ROOM.name,
|
||||
track_id: 'TR_aLiceMic0pus',
|
||||
output: { file: { filepath: 'tracks/alice-microphone.ogg' } }
|
||||
}
|
||||
},
|
||||
outputs: [
|
||||
{ kind: 'file', filename: 'tracks/alice-microphone.ogg', startedAt: failedStart, endedAt: failedEnd }
|
||||
],
|
||||
|
|
@ -94,6 +112,15 @@ export const egresses: EgressDocument[] = [
|
|||
startedAt: at('10:05:00'),
|
||||
updatedAt: at('10:05:03'),
|
||||
request: { layout: 'grid', preset: 'H264_1080P_30', outputs: ['stream', 'segments'] },
|
||||
rawRequest: {
|
||||
room_composite: {
|
||||
room_name: ACTIVE_ROOM.name,
|
||||
layout: 'grid',
|
||||
options: { preset: 'H264_1080P_30' },
|
||||
stream_outputs: [{ protocol: 'RTMP', urls: ['rtmps://live.example.com/app/***'] }],
|
||||
segment_outputs: [{ playlist_name: 'webinar-q4/playlist.m3u8' }]
|
||||
}
|
||||
},
|
||||
outputs: [
|
||||
{ kind: 'stream', url: 'rtmps://live.example.com/app/***', status: 'ACTIVE', startedAt: at('10:05:03') },
|
||||
{
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import {
|
|||
unixNs,
|
||||
unixSeconds
|
||||
} from './scenario.js';
|
||||
import { recordingRequest } from './egress-ingress.js';
|
||||
|
||||
/**
|
||||
* One stored raw event of every type of the default `events.store_types`. Payloads follow protojson naming, nest the set
|
||||
|
|
@ -100,15 +101,6 @@ function rtpStats(start: Date, end: Date, packets: number, bytes: number, lost:
|
|||
};
|
||||
}
|
||||
|
||||
const recordingRequest = {
|
||||
room_composite: {
|
||||
room_name: CLOSED_ROOM.name,
|
||||
layout: 'speaker',
|
||||
options: { preset: 'H264_720P_30' },
|
||||
file_outputs: [{ file_type: 'MP4', filepath: 'recordings/daily-standup-2026-10-12.mp4' }]
|
||||
}
|
||||
};
|
||||
|
||||
const recordingEgress = {
|
||||
egress_id: EG.recording,
|
||||
room_id: CLOSED_ROOM.sid,
|
||||
|
|
|
|||
|
|
@ -280,6 +280,9 @@ const egresses = obj({
|
|||
{ description: 'Sanitized request: no credentials, no stream keys' }
|
||||
)
|
||||
),
|
||||
rawRequest: anyObject(
|
||||
'Request in raw event payload encoding: the set member of the request oneof, allow-listed fields'
|
||||
),
|
||||
outputs: req(
|
||||
arr(
|
||||
obj({
|
||||
|
|
|
|||
|
|
@ -13,6 +13,14 @@
|
|||
"endedAt": { "$date": "2026-10-12T09:40:20.000Z" },
|
||||
"durationMs": { "$numberLong": "2300000" },
|
||||
"request": { "layout": "speaker", "preset": "H264_720P_30", "outputs": ["file"] },
|
||||
"rawRequest": {
|
||||
"room_composite": {
|
||||
"room_name": "Daily-Standup",
|
||||
"layout": "speaker",
|
||||
"options": { "preset": "H264_720P_30" },
|
||||
"file_outputs": [{ "file_type": "MP4", "filepath": "recordings/daily-standup-2026-10-12.mp4" }]
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"kind": "file",
|
||||
|
|
@ -51,6 +59,13 @@
|
|||
"error": "upload failed: access denied",
|
||||
"errorCode": { "$numberInt": "403" },
|
||||
"request": { "trackId": "TR_aLiceMic0pus", "outputs": ["file"] },
|
||||
"rawRequest": {
|
||||
"track": {
|
||||
"room_name": "Daily-Standup",
|
||||
"track_id": "TR_aLiceMic0pus",
|
||||
"output": { "file": { "filepath": "tracks/alice-microphone.ogg" } }
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"kind": "file",
|
||||
|
|
@ -79,6 +94,15 @@
|
|||
"startedAt": { "$date": "2026-10-12T10:05:00.000Z" },
|
||||
"updatedAt": { "$date": "2026-10-12T10:05:03.000Z" },
|
||||
"request": { "layout": "grid", "preset": "H264_1080P_30", "outputs": ["stream", "segments"] },
|
||||
"rawRequest": {
|
||||
"room_composite": {
|
||||
"room_name": "Webinar-Q4",
|
||||
"layout": "grid",
|
||||
"options": { "preset": "H264_1080P_30" },
|
||||
"stream_outputs": [{ "protocol": "RTMP", "urls": ["rtmps://live.example.com/app/***"] }],
|
||||
"segment_outputs": [{ "playlist_name": "webinar-q4/playlist.m3u8" }]
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"kind": "stream",
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@
|
|||
"additionalProperties": false,
|
||||
"description": "Sanitized request: no credentials, no stream keys"
|
||||
},
|
||||
"rawRequest": {
|
||||
"bsonType": "object",
|
||||
"description": "Request in raw event payload encoding: the set member of the request oneof, allow-listed fields"
|
||||
},
|
||||
"outputs": {
|
||||
"bsonType": "array",
|
||||
"items": {
|
||||
|
|
|
|||
|
|
@ -318,6 +318,12 @@ export interface EgressDocument extends EntityBase {
|
|||
error?: string;
|
||||
errorCode?: number;
|
||||
request: EgressRequestSummary;
|
||||
/**
|
||||
* The request as raw event payloads store `egress.request`: the set member of the request oneof under its protocol
|
||||
* name (`room_composite`, `web`, `participant`, `track_composite`, `track`, `egress`, `replay`), allow-listed fields
|
||||
* only.
|
||||
*/
|
||||
rawRequest?: Record<string, unknown>;
|
||||
outputs: EgressOutput[];
|
||||
/** Status transitions in order; at most 50 entries. */
|
||||
statusHistory: EgressStatusChange[];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
schema: spec-driven
|
||||
created: 2026-10-27
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Design
|
||||
|
||||
## Context
|
||||
|
||||
- The codec encodes a protocol message through the allow-list: protocol field names, enum values as names, the set
|
||||
member of a oneof as a document named after the oneof, URLs redacted by rule. Raw events store `EgressInfo` this
|
||||
way, so `payload.egress.request` holds the full request without secrets.
|
||||
- The egress projection writes `request` from `egressRequest()`, a summary built field by field.
|
||||
- Validators are applied at every start with `validationLevel: strict` and the configured action (`warn` by
|
||||
default).
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:**
|
||||
- The egress document keeps the full request, without secrets, for the entities retention.
|
||||
|
||||
**Non-Goals:**
|
||||
- Replacing the summary, which the list and the detail facts read.
|
||||
- Backfilling stored documents.
|
||||
|
||||
## Decisions
|
||||
|
||||
### Same encoding as the raw payloads
|
||||
|
||||
`rawRequest` is the `request` member of `EgressInfo` as the codec encodes it: one key, the oneof member that is set
|
||||
(`room_composite`, `web`, `participant`, `track_composite`, `track`, `egress` or `replay`), holding the request. It is
|
||||
the value `payload.egress.request` of the raw events of the same egress. The allow-list stays the only place that
|
||||
decides what is stored.
|
||||
|
||||
Alternative rejected: unwrapping the oneof member. The member tells the legacy room composite API apart from the
|
||||
unified `StartEgress` and `ExportReplay` APIs, which `type` does not.
|
||||
|
||||
Alternative rejected: protojson with camelCase names. Raw payloads and this field would show the same request in two
|
||||
namings.
|
||||
|
||||
### Written on every event
|
||||
|
||||
Every `EGRESS_*` event carries the request, so the field is set on each upsert, like `request`. It is absent when the
|
||||
event has no request.
|
||||
|
||||
### Data model
|
||||
|
||||
- Collection `egresses`: new optional field `rawRequest` (object). No index; the field is never queried.
|
||||
- Retention: the entities retention of the document.
|
||||
- Schema version: unchanged (1). The field is optional, and the contract has not shipped in a release. A dashboard
|
||||
that does not know the field ignores it.
|
||||
- Migration from legacy events: the legacy migration decodes the legacy `EgressInfo` and projects it like a live
|
||||
event, so a migrated egress gets `rawRequest` when its legacy events carry a request.
|
||||
- Performance: one encode of a message the codec already encodes for the raw event. The document grows by the size of
|
||||
the request, usually under 1 KB.
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
- [A future allow-list entry stores a secret in the document too] → The codec tests keep checking every allow-listed
|
||||
field against the secret patterns, and an edition that shows the field can filter it again on read.
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
# Proposal
|
||||
|
||||
## Why
|
||||
|
||||
The `request` of an egress document is a summary: source, encoding and the kinds of output. The configuration of each
|
||||
output (file type and path, stream protocol and URLs, segment and image settings), the audio routes of the unified API
|
||||
and the replay offsets are lost. The full request, through the secrets allow-list, exists only in the raw `EGRESS_*`
|
||||
event payloads, which expire with the events retention while the egress document keeps the entities retention.
|
||||
|
||||
## What Changes
|
||||
|
||||
- The `egresses` document gains an optional `rawRequest`: the request of the egress encoded as raw event payloads
|
||||
encode `egress.request`, with the protocol field names, the set member of each oneof under the oneof name and only
|
||||
the allow-listed fields. Credentials, storage uploads and webhooks are never stored, and URLs are redacted.
|
||||
- `request` keeps its summary, unchanged.
|
||||
|
||||
Scope excludes the following:
|
||||
- Egress documents already stored, which have no `rawRequest` until their egress receives another event.
|
||||
- Any change to the allow-list or to the request summary.
|
||||
- How an edition shows the field.
|
||||
|
||||
Edition: CE and PRO (data model). Repositories: dashboard (contract, typings, data generator, specs) and
|
||||
openvidu-livekit (projection and document types).
|
||||
|
||||
CHANGELOG: none. The analytics data model has not shipped in a release.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
|
||||
None.
|
||||
|
||||
### Modified Capabilities
|
||||
|
||||
- `data-model`: the egress document stores the request in raw payload encoding.
|
||||
|
||||
## Impact
|
||||
|
||||
- Contract (`dashboard-ce/typings`): `EgressDocument.rawRequest`, the `egresses` validator and the egress fixtures; the
|
||||
contract version stays 1.
|
||||
- Writer: the egress projection, the `EgressDocument` struct, the vendored contract and the integration tests.
|
||||
- Data generator: generated egress documents carry `rawRequest`.
|
||||
- Dashboard backend: none; the egress list projection does not read the field.
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Egress document
|
||||
Edition: CE and PRO. An `egresses` document SHALL describe one egress: room reference, type, source,
|
||||
lifecycle status with history, timestamps, error, a sanitized request summary, the request in raw payload encoding and
|
||||
its outputs, with the field list of the shared contract. `roomNameLower` SHALL be the lower-cased `roomName`, written
|
||||
together with it. `rawRequest` SHALL hold the request as the raw events of the egress store `egress.request`: the
|
||||
member of the request oneof that is set, under its protocol name, through the secrets allow-list.
|
||||
|
||||
#### Scenario: Fields of an egress document
|
||||
- **WHEN** an egress document is read
|
||||
- **THEN** it has `roomSid`, `roomName`, `roomNameLower`, `type` (`room_composite` | `web` | `participant` | `track_composite` | `track`), `sourceType`, `status` (`starting` | `active` | `ending` | `complete` | `failed` | `aborted` | `limit_reached`), `closedBy`, `startedAt`, `updatedAt`, `endedAt`, `durationMs`, `error`, `errorCode`, `request` (sanitized), `rawRequest?`, `outputs[] { kind, name or url, location, size, durationMs, status, error, startedAt, endedAt }`, `statusHistory[]` with at most 50 entries, `retryCount`, `backupStorageUsed`, `schemaVersion`, `expiresAt`
|
||||
|
||||
#### Scenario: Egress completes
|
||||
- **GIVEN** `EGRESS_STARTED`, two `EGRESS_UPDATED` and `EGRESS_ENDED` with status COMPLETE and one file result
|
||||
- **WHEN** all are processed
|
||||
- **THEN** the document has `status: "complete"`, one output of kind `file` with its filename, location, size and duration, and four `statusHistory` entries
|
||||
|
||||
#### Scenario: Credentials are never stored
|
||||
- **GIVEN** an egress request with S3 credentials and a stream URL containing a key
|
||||
- **WHEN** the egress is projected
|
||||
- **THEN** neither `request` nor `rawRequest` contains a credential field, and the stream output URL has its key redacted
|
||||
|
||||
#### Scenario: Raw request
|
||||
- **GIVEN** a room composite egress of room `Demo-Room` with layout `grid`, preset `H264_720P_30`, an MP4 file output with path `recordings/demo.mp4` uploaded to S3 with credentials, and a stream output to `rtmps://live.example.com/app/live_123?token=abc`
|
||||
- **WHEN** its `EGRESS_STARTED` event is processed
|
||||
- **THEN** `rawRequest` is `{ room_composite: { room_name: "Demo-Room", layout: "grid", options: { preset: "H264_720P_30" }, file_outputs: [{ file_type: "MP4", filepath: "recordings/demo.mp4" }], stream_outputs: [{ urls: ["rtmps://live.example.com/app/***"] }] } }`, the value of `payload.egress.request` in the stored event
|
||||
|
||||
#### Scenario: Room name search field
|
||||
- **GIVEN** an egress of room `Demo-Room`
|
||||
- **WHEN** its document is read
|
||||
- **THEN** it has `roomName: "Demo-Room"` and `roomNameLower: "demo-room"`
|
||||
|
||||
#### Scenario: Web egress without room
|
||||
- **GIVEN** a web egress, which has no room
|
||||
- **WHEN** its document is read
|
||||
- **THEN** it has neither `roomName` nor `roomNameLower`
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Tasks
|
||||
|
||||
## 1. Typings
|
||||
|
||||
- [x] 1.1 Add `rawRequest?: Record<string, unknown>` to `EgressDocument` in `typings/src/data-model/entities.ts` and the
|
||||
`rawRequest` object to the `egresses` schema in `typings/contract/schemas.ts`. Add the raw request of each egress
|
||||
fixture in `contract/fixtures/egress-ingress.ts`, shared with the recording event of `contract/fixtures/events.ts`.
|
||||
Regenerate the contract with `pnpm --filter @openvidu-dashboard/typings run contract`. Verify with the typings
|
||||
typecheck and tests.
|
||||
|
||||
## 2. Writer (openvidu-livekit)
|
||||
|
||||
- [x] 2.1 Vendor the contract with `typings/contract/vendor.sh`. Add `RawRequest` to the `EgressDocument` struct and set
|
||||
`rawRequest` in the egress projection from the `request` member of the encoded `EgressInfo`. Verify with the unit
|
||||
tests of `openvidu/analytics/...` and the lint of the fork.
|
||||
- [x] 2.2 Assert in the egress integration tests that `rawRequest` holds the file and stream outputs, with the stream
|
||||
URL redacted and no credential. Verify with the analytics integration tests and `typings/contract/writer-check.sh`.
|
||||
|
||||
## 3. Data generator
|
||||
|
||||
- [x] 3.1 Write `rawRequest` on every generated egress document, from the request the generator derives for the egress
|
||||
events. Verify with the data generator tests and by loading a demo dataset.
|
||||
|
||||
## 4. Docs and CHANGELOG
|
||||
|
||||
- [x] 4.1 No CHANGELOG line: the analytics data model has not shipped. Update the `AGENTS.md` files only if a command or
|
||||
path changes. Verify by reading the change against them.
|
||||
|
||||
## 5. Integration check
|
||||
|
||||
- [x] 5.1 Run `./dashboard.sh check` and the backend integration tests. Verify that they pass.
|
||||
Loading…
Reference in New Issue