diff --git a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts index 68f4946d2..da603b3d4 100644 --- a/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts +++ b/openvidu-dashboard/dashboard-ce/tools/data-generator/src/derive.ts @@ -1078,8 +1078,12 @@ function toDoc(scope: string, bucket: number, b: Bucket, daily: boolean): Rollup const BROWSER_KEYS = new Set(['chrome', 'firefox', 'safari', 'edge']); -/** Rollup key of a client browser: the lower-case name of a known browser, else `other`. */ +/** Rollup key of a client browser: `none` without one, the lower-case name of a known browser, else `other`. */ function browserKey(browser: string): string { + if (browser === '') { + return 'none'; + } + const key = browser.toLowerCase(); return BROWSER_KEYS.has(key) ? key : 'other'; diff --git a/openvidu-dashboard/dashboard-ce/typings/contract/fixtures/stats-and-rollups.ts b/openvidu-dashboard/dashboard-ce/typings/contract/fixtures/stats-and-rollups.ts index 29b5e7509..f76203647 100644 --- a/openvidu-dashboard/dashboard-ce/typings/contract/fixtures/stats-and-rollups.ts +++ b/openvidu-dashboard/dashboard-ce/typings/contract/fixtures/stats-and-rollups.ts @@ -203,7 +203,7 @@ export const analyticsDaily: DailyRollupDocument[] = [ byKind: { STANDARD: 6, INGRESS: 2, EGRESS: 2, SIP: 1, AGENT: 1 }, bySdk: { JS: 5, ANDROID: 1 }, byOs: { windows: 1, macos: 3, linux: 1, android: 1 }, - byBrowser: { chrome: 2, safari: 2, firefox: 1, other: 1 }, + byBrowser: { chrome: 2, safari: 2, firefox: 1, none: 1 }, byConnectionType: { udp: 2, tcp: 1, turn: 1 }, disconnectReasons: { CLIENT_INITIATED: 1, diff --git a/openvidu-dashboard/dashboard-ce/typings/fixtures/analytics_daily.json b/openvidu-dashboard/dashboard-ce/typings/fixtures/analytics_daily.json index 1e0843495..fce7b59c7 100644 --- a/openvidu-dashboard/dashboard-ce/typings/fixtures/analytics_daily.json +++ b/openvidu-dashboard/dashboard-ce/typings/fixtures/analytics_daily.json @@ -33,7 +33,7 @@ "chrome": { "$numberInt": "2" }, "safari": { "$numberInt": "2" }, "firefox": { "$numberInt": "1" }, - "other": { "$numberInt": "1" } + "none": { "$numberInt": "1" } }, "byConnectionType": { "udp": { "$numberInt": "2" }, diff --git a/openvidu-dashboard/dashboard-ce/typings/schema/analytics_daily.schema.json b/openvidu-dashboard/dashboard-ce/typings/schema/analytics_daily.schema.json index 98d478e81..021f8c2d8 100644 --- a/openvidu-dashboard/dashboard-ce/typings/schema/analytics_daily.schema.json +++ b/openvidu-dashboard/dashboard-ce/typings/schema/analytics_daily.schema.json @@ -91,6 +91,7 @@ "firefox": { "bsonType": ["int", "long"] }, "safari": { "bsonType": ["int", "long"] }, "edge": { "bsonType": ["int", "long"] }, + "none": { "bsonType": ["int", "long"] }, "other": { "bsonType": ["int", "long"] } }, "additionalProperties": false diff --git a/openvidu-dashboard/dashboard-ce/typings/schema/analytics_hourly.schema.json b/openvidu-dashboard/dashboard-ce/typings/schema/analytics_hourly.schema.json index bc711e405..cb3e5f9a8 100644 --- a/openvidu-dashboard/dashboard-ce/typings/schema/analytics_hourly.schema.json +++ b/openvidu-dashboard/dashboard-ce/typings/schema/analytics_hourly.schema.json @@ -91,6 +91,7 @@ "firefox": { "bsonType": ["int", "long"] }, "safari": { "bsonType": ["int", "long"] }, "edge": { "bsonType": ["int", "long"] }, + "none": { "bsonType": ["int", "long"] }, "other": { "bsonType": ["int", "long"] } }, "additionalProperties": false diff --git a/openvidu-dashboard/dashboard-ce/typings/src/data-model/enums.ts b/openvidu-dashboard/dashboard-ce/typings/src/data-model/enums.ts index 0b7b1aaf0..0ebd8ce04 100644 --- a/openvidu-dashboard/dashboard-ce/typings/src/data-model/enums.ts +++ b/openvidu-dashboard/dashboard-ce/typings/src/data-model/enums.ts @@ -127,12 +127,13 @@ export const OsFamily = { } as const; export type OsFamily = ValueOf; -/** Fixed vocabulary of rollup browser keys; values outside it are counted as `other`. */ +/** Fixed vocabulary of rollup browser keys; a session that reports no browser is `none`, other values `other`. */ export const BrowserFamily = { CHROME: 'chrome', FIREFOX: 'firefox', SAFARI: 'safari', EDGE: 'edge', + NONE: 'none', OTHER: 'other' } as const; export type BrowserFamily = ValueOf; diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/.openspec.yaml b/openvidu-dashboard/openspec/changes/writer-review-fixes/.openspec.yaml new file mode 100644 index 000000000..ac359ee85 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-03 diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/design.md b/openvidu-dashboard/openspec/changes/writer-review-fixes/design.md new file mode 100644 index 000000000..70a05bba2 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/design.md @@ -0,0 +1,85 @@ +# Design + +## Context + +- **Ingress state:** `IOInfoService.UpdateIngressState` (`pkg/service/ioservice_ingress.go`) stores the state the + ingress service sends and emits `IngressStarted`, `IngressUpdated` or `IngressEnded` with it. On a local deployment + the buffering and publishing states arrive with an empty `room_id`, and only the ending state names the room + (`RM_rgFu9jsQne28`), although the ingress participant joined that room 40 ms before the publishing state. +- **Ingress projection:** the writer's `projection/ingress.go` lifts `roomSid` from `state.room_id`, and adds the + session to the room's `internal.ingressSessionIds` only when it is set. +- **Ingress store:** `IOInfoService` holds an `IngressStore`, which `getIngressStore` resolves to the `*RedisStore`, + also the `ObjectStore` that loads rooms by name. Ingress needs Redis. +- **Egress history:** `projection/egress.go` adds `{ status, at }` to `statusHistory` with `$setUnion`, which drops only + identical entries: two `active` updates 13 ms apart, one per output, both stay. +- **Browser vocabulary:** `projection/vocabulary.go` maps an empty browser to `other`, through `browserFamily()`. + +## Goals / Non-Goals + +**Goals:** +- A publishing ingress session is linked to, and counted by, its room with no new projection path. +- The contract stays additive. + +**Non-Goals:** +- Changing how the ingress service reports its state (upstream LiveKit ingress). + +## Decisions + +### D1. The server names the room of an ingress state + +In an OPENVIDU block of `UpdateIngressState`, when `req.State.RoomId` is empty and the ingress has a room name, the +server loads that room through the store and sets the room's SID on the state it emits. A missing or failing load +leaves it empty, logged at debug. The block runs before the telemetry calls, so the webhook and the analytics event +carry the same state, and the stored state is updated with it too. + +The store is reached through a type assertion of `s.is` to the room loader that `*RedisStore` implements, so the wire +graph is unchanged. + +Rejected: linking the session in the writer when the ingress participant joins. That event has no `resource_id`, so +the room could not count the session until it ends, and the room's counters would still miss a live ingress. + +### D2. One status history entry per egress status + +The `statusHistory` update keeps, for the incoming status, the entry with the earliest `at`: it drops any stored entry +of the same status whose `at` is later, adds the incoming entry when no entry of that status is left, sorts by `at` and +keeps the last 50. Egress statuses only move forward, so a repeated status is never a new step. + +Rejected: comparing with the last entry only, which depends on the order events arrive in. + +### D3. `none` in the browser vocabulary + +`browserFamily("")` returns `none`, and `BrowserFamily` gains `NONE: 'none'`, in the contract that the validator and +the writer share. + +Rejected: leaving `other` for both, which hides SDKs without a browser among unknown browsers. + +## Data model + +| Collection | Change | +| --- | --- | +| `ingress_sessions` | `roomSid` set from the first publishing state, instead of the ending one. | +| `rooms` | `internal.ingressSessionIds`, and the counters it feeds, set as soon as the session publishes. | +| `egresses` | `statusHistory` with one entry per status. | +| `analytics_5min`, `analytics_hourly`, `analytics_daily` | `participants.byBrowser.none`. | + +- No index change. +- No schema version bump: the v2 schema has not shipped. +- Legacy migration: the legacy events carry no ingress state of this kind and no output updates, so the mapping is + unchanged. Legacy sessions without a browser stay under `other`. + +## Performance budget + +One `LoadRoom` (a Redis `HGET`) per ingress state update without a room id: a handful per ingress session. The writer +budgets are unchanged; the batch benchmark covers the egress update. + +## Risks / Trade-offs + +- [The room is recreated under the same name while the ingress publishes] → The state names the room active at the + time of the update, the one the ingress participant is in. +- [The webhooks of those state updates now carry `room_id`] → An addition, matching what the ending state already + sends. + +## Migration Plan + +Ships with the 3.10 server, and the dashboard typings ship with it through the contract. `ui-review-fixes` labels the +new key. diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/proposal.md b/openvidu-dashboard/openspec/changes/writer-review-fixes/proposal.md new file mode 100644 index 000000000..05fee7d4e --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/proposal.md @@ -0,0 +1,51 @@ +# Proposal + +## Why + +A review against a real deployment found three places where the analytics data does not match what happened: +- **Ingress session room:** the ingress service reports no `room_id` while an ingress session publishes, only when + it ends, so a live session has no room. The room's ingress section, the session's Room column and the room's ingress + counters miss it until the stream stops. +- **Egress status history:** an egress with several outputs sends one update per output that starts, and the status + history keeps both, so the dashboard shows "Active → Active". +- **Sessions without a browser:** sessions of SDKs that report no browser, such as the Go SDK, are counted with unknown + browsers under `other`, so the browser breakdown cannot say "No browser". + +## What Changes + +- The server fills the room id of an ingress state update that reports none from the active room of the ingress's + room name, before its telemetry and webhooks. The writer then links the session to its room, and counts it there, + as soon as it publishes. +- The writer keeps one status history entry per egress status, at the first time it was reported. +- The rollup browser vocabulary gains `none`, counted for a session that reports no browser; `other` keeps the + browsers outside the vocabulary. + +Scope excludes the following: +- Backfilling sessions, histories or rollups written before the change. +- The dashboard labels of `none` (change `ui-review-fixes`). + +Edition: CE and PRO (the writer and the server code every edition runs). Repositories: openvidu-livekit +(`pkg/service/ioservice_ingress.go`, `openvidu/analytics/projection`) and the dashboard (typings contract, specs). + +CHANGELOG lines: none. The v2 data model and its writer have not shipped in a release. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `server-analytics-writer`: ingress sessions get their room while publishing, and empty browsers count as `none`. +- `data-model`: one status history entry per egress status. + +## Impact + +- Server: an OPENVIDU block in `IOInfoService.UpdateIngressState` that loads the room by name when the state has no + room id. +- Writer: the egress status history update and the browser vocabulary in `projection/`, with unit and integration + tests. +- Typings: `BrowserFamily.NONE` in `src/data-model/enums.ts`, the regenerated `schema/`, the vendored contract and + `./dashboard.sh contract-check`. +- No schema version bump: the v2 schema has not shipped. diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/data-model/spec.md b/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/data-model/spec.md new file mode 100644 index 000000000..80dc5a263 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/data-model/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: Egress document +Edition: CE and PRO. An `egresses` document SHALL describe one egress: room reference, type, source, +lifecycle status with history, timestamps, error, a sanitized request summary, the request in raw payload encoding and +its outputs, with the field list of the shared contract. `statusHistory` SHALL hold one entry per status, at its first +report. `roomNameLower` SHALL be the lower-cased `roomName`, written together with it. + +#### Scenario: Fields of an egress document +- **WHEN** an egress document is read +- **THEN** it has `roomSid`, `roomName`, `roomNameLower`, `type` (`room_composite` | `web` | `participant` | `track_composite` | `track`), `sourceType`, `status` (`starting` | `active` | `ending` | `complete` | `failed` | `aborted` | `limit_reached`), `closedBy`, `startedAt`, `updatedAt`, `endedAt`, `durationMs`, `error`, `errorCode`, `request` (sanitized), `rawRequest?`, `outputs[] { kind, name or url, location, size, durationMs, status, error, startedAt, endedAt }`, `statusHistory[]` with at most 50 entries, `retryCount`, `backupStorageUsed`, `schemaVersion`, `expiresAt` + +#### Scenario: Egress completes +- **GIVEN** `EGRESS_STARTED`, two `EGRESS_UPDATED` and `EGRESS_ENDED` with status COMPLETE and one file result +- **WHEN** all are processed +- **THEN** the document has `status: "complete"`, one output of kind `file` with its filename, location, size and duration, and four `statusHistory` entries + +#### Scenario: Status reported twice +- **GIVEN** an egress with a file and a stream output whose two `EGRESS_UPDATED` report `active` at 09:06:49.336 and + 09:06:49.349 +- **WHEN** they are processed, in either order +- **THEN** `statusHistory` holds one `active` entry, at 09:06:49.336 + +#### Scenario: Credentials are never stored +- **GIVEN** an egress request with S3 credentials and a stream URL containing a key +- **WHEN** the egress is projected +- **THEN** neither `request` nor `rawRequest` contains a credential field, and the stream output URL has its key redacted + +#### Scenario: Room name search field +- **GIVEN** an egress of room `Demo-Room` +- **WHEN** its document is read +- **THEN** it has `roomName: "Demo-Room"` and `roomNameLower: "demo-room"` + +#### Scenario: Web egress without room +- **GIVEN** a web egress, which has no room +- **WHEN** its document is read +- **THEN** it has neither `roomName` nor `roomNameLower` diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/server-analytics-writer/spec.md b/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/server-analytics-writer/spec.md new file mode 100644 index 000000000..6aa845111 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/specs/server-analytics-writer/spec.md @@ -0,0 +1,72 @@ +## MODIFIED Requirements + +### Requirement: Lifted keys and room resolution +Edition: CE and PRO. Every document SHALL carry lifted references resolved from the event: `ROOM_CREATED` +from `room.sid`, egress events from `egress.room_id` and `egress.room_name`, ingress events from +`ingress.state.room_id`, which the server fills from the active room of `ingress.room_name` when the ingress reports +none, and `ingress.room_name`, track events from the minimal `room {sid, name}`; a missing room leaves the reference +absent. + +#### Scenario: ROOM_CREATED without room_id +- **GIVEN** a `ROOM_CREATED` event with empty `room_id`, `room.sid = "RM_x"` and `room.creation_time_ms` +- **WHEN** it is projected +- **THEN** the `rooms` document has `_id: "RM_x"` and `createdAt` from `creation_time_ms`, and the raw event has `roomSid: "RM_x"` + +#### Scenario: Egress before any room event +- **GIVEN** an `EGRESS_STARTED` with `egress.room_id` and `egress.room_name` and no prior room event +- **WHEN** it is projected +- **THEN** the `egresses` document and the raw event carry `roomSid` and `roomName`, and the `rooms` document is upserted with `features.egress: true` and its egress counter + +#### Scenario: Web egress without room +- **GIVEN** a web egress with empty `room_id` +- **WHEN** it is projected +- **THEN** `roomSid` is absent from the egress document and the raw event, and no `rooms` write model is produced + +#### Scenario: Ingress session key +- **GIVEN** an `INGRESS_STARTED` with `ingress.state.resource_id` +- **WHEN** it is projected +- **THEN** the `ingress_sessions` document has `_id` equal to `resource_id`, `ingressId` and `roomSid` from `state.room_id`, and the raw event has `ingressSessionId` + +#### Scenario: Ingress publishing before its state names the room +- **GIVEN** an ingress of room `qa-ingress`, active as `RM_x`, whose service sends the publishing state without + `room_id` +- **WHEN** the server handles the state update and the writer projects its `INGRESS_STARTED` +- **THEN** the event carries `state.room_id: "RM_x"`, the `ingress_sessions` document has `roomSid: "RM_x"` while it + publishes, and the room counts the session as active + +#### Scenario: Ingress of a room that is not active +- **GIVEN** an ingress whose state has no `room_id` and whose room name has no active room +- **WHEN** the server handles the state update +- **THEN** the state keeps no `room_id` and the session document has no `roomSid` until an event names the room + +#### Scenario: Subscriber identity resolved +- **GIVEN** a participant session `PA_1` already projected with identity `alice` +- **WHEN** a `TRACK_SUBSCRIBED` with `participant {sid: "PA_1"}` and `publisher {sid, identity}` is processed +- **THEN** the subscription has `subscriberIdentity: "alice"`, `publisherSid` and `publisherIdentity` from `publisher` + +### Requirement: Rollup counters from events +Edition: CE and PRO. Each projected event SHALL increment the hourly and daily counters of its UTC bucket +in both scopes, and its 5-minute lifecycle counters, once: an event stored raw counts only when its raw insert +succeeds, so an event already stored by a previous process never counts again; an event not stored raw counts when +it is projected; an in-process retry never reapplies a rollup step that succeeded. Map keys SHALL be enum names or a +fixed vocabulary, never user input. + +#### Scenario: Batch retried after a transient error +- **GIVEN** a batch whose raw inserts succeeded and whose rollup step failed with a network error +- **WHEN** the batch is retried +- **THEN** raw inserts are skipped as duplicates, the rollup step is applied once and the counters equal the event counts + +#### Scenario: Replayed events from another process +- **GIVEN** events already stored by a previous process +- **WHEN** they are processed again +- **THEN** the rollup counters are unchanged + +#### Scenario: Vocabulary keys +- **GIVEN** a `PARTICIPANT_JOINED` with `client_info.sdk: JS`, `os: "Windows 11"` and `browser: "Chrome 130"` +- **WHEN** it is projected +- **THEN** the hourly rollup increments `participants.bySdk.JS`, `participants.byOs.windows` and `participants.byBrowser.chrome`, and a value outside the vocabulary increments `other` + +#### Scenario: Session without a browser +- **GIVEN** a `PARTICIPANT_JOINED` with `client_info.sdk: GO` and an empty `browser` +- **WHEN** it is projected +- **THEN** the hourly rollup increments `participants.byBrowser.none`, not `participants.byBrowser.other` diff --git a/openvidu-dashboard/openspec/changes/writer-review-fixes/tasks.md b/openvidu-dashboard/openspec/changes/writer-review-fixes/tasks.md new file mode 100644 index 000000000..15267cf85 --- /dev/null +++ b/openvidu-dashboard/openspec/changes/writer-review-fixes/tasks.md @@ -0,0 +1,26 @@ +# Tasks + +## 1. Typings + +- [x] 1.1 Add `NONE: 'none'` to `BrowserFamily`, regenerate `schema/` and `fixtures/`, and vendor the contract into + `openvidu/analytics/schema/contract/` with `contract/vendor.sh`. Verify with the typings contract tests and + `./dashboard.sh contract-check ../openvidu-livekit`. + +## 2. Server and writer (openvidu-livekit) + +- [x] 2.1 Fill the empty `room_id` of an ingress state from the room of its room name in `UpdateIngressState` (OPENVIDU + block). Verify with a unit test of `IOInfoService` with an active room, a missing room and a state that names its + room. +- [x] 2.2 Keep one `statusHistory` entry per egress status at its earliest `at`. Verify with projection unit tests and a + writer integration test of "Status reported twice" in both orders. +- [x] 2.3 Map an empty browser to `none` in `browserFamily()`. Verify with a vocabulary unit test and a writer + integration test of "Session without a browser". +- [x] 2.4 Lint the fork with the Dockerfile Go toolchain and run its unit and integration tests. Verify that both pass. + +## 3. Integration check + +- [ ] 3.1 Run `./dashboard.sh check` and `./dashboard.sh contract-check ../openvidu-livekit`. Verify that both pass. +- [ ] 3.2 On a local deployment with the nightly server image, publish an RTMP ingress into an active room and start a + room composite egress with a file and a stream output. Verify that the session has `roomSid` while it publishes, + the room counts it, the egress history reads starting, active, ending, complete, and a Go SDK session is counted + under `byBrowser.none`.