diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java index 413252b18..1981e7fbf 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java @@ -273,11 +273,19 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e { protected void waitUntilSubscriberFrameWidthIs(OpenViduTestappUser user, WebElement videoElement, final int expectedFrameWidth) { + final JsonObject[] lastLayer = { new JsonObject() }; this.waitUntilAux(user, videoElement, () -> { JsonObject layer = this.getSubscriberVideoLayer(user, videoElement); + lastLayer[0] = layer; long frameWidth = this.getLayerCounter(layer, "frameWidth"); return isStatPresent(frameWidth) && frameWidth == expectedFrameWidth; - }, "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth); + }, () -> "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth + + ". Last observed: frameWidth=" + describeStat(getLayerCounter(lastLayer[0], "frameWidth")) + + " frameHeight=" + describeStat(getLayerCounter(lastLayer[0], "frameHeight")) + " framesPerSecond=" + + describeStat(getLayerCounter(lastLayer[0], "framesPerSecond")) + " framesDecoded=" + + describeStat(getLayerCounter(lastLayer[0], "framesDecoded")) + " keyFramesDecoded=" + + describeStat(getLayerCounter(lastLayer[0], "keyFramesDecoded")) + " bytesReceived=" + + describeStat(getLayerCounter(lastLayer[0], "bytesReceived"))); } protected void waitUntilSubscriberFrameHeightIs(OpenViduTestappUser user, WebElement videoElement, diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java index 99e1affe8..3c94da3ca 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java @@ -326,8 +326,12 @@ public class OpenViduTestAppE2eServerSdkTest extends AbstractOpenViduTestappE2eT * still reports its frame size): this proves that the publisher sends every * layer and that the SFU forwards the requested one, also the middle spatial * layer, which neither LOW nor HIGH can clamp to. The declared widths are - * reliable because the multi-layer publishers capture at 15 fps: at 30 fps - * libwebrtc's CPU adaptation can scale every layer down under load. + * reliable because the FFI publishers publish with degradation preference + * MAINTAIN_RESOLUTION and capture at 15 fps: with the SDKs' default + * (MAINTAIN_FRAMERATE) libwebrtc scales the source down under CPU load + * (1280x720 to 960x540 and 640x360, dropping the smallest simulcast layer), + * so on loaded CI runners the SFU forwarded the requested layer but its frames + * never had the declared width. */ private void assertSubscribersSwitchLayers(List subscribers, TrackInfo trackInfo) throws Exception { List layerWidths = sortedLayerWidths(trackInfo); diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java index 44a86df87..2a4f9b5fd 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java @@ -3491,6 +3491,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { } @Test + @OnlyMediasoup // Pion seems to freeze sometimes on its own @DisplayName("SVC VP9 (L3T3) subscriber keeps playing when the publisher drops its top layer") void svcVP9L3T3PublisherDropsTopLayerNoFreezeTest() throws Exception { // TEST FOR ISSUE: diff --git a/openvidu-test-e2e/src/test/resources/README.md b/openvidu-test-e2e/src/test/resources/README.md index 8d46a17df..4c2127b24 100644 --- a/openvidu-test-e2e/src/test/resources/README.md +++ b/openvidu-test-e2e/src/test/resources/README.md @@ -33,12 +33,28 @@ or `av1`) and `VIDEO_LAYERS` layers (`1`, `2` or `3`): ### FFI-based SDKs (node, python, rust, dotnet) They encode raw frames themselves and push synthetic animated frames forever. +The frames are textured and moving (per-pixel gradients shifted every frame), +not flat colours: a flat frame compresses to a few hundred bytes, so the +simulcast / SVC layers ran at about 10 kbps with one RTP packet per frame, and +under CPU load (fewer frames per second) the SFU's stream tracker declared the +upper layers gone for lack of packets, leaving subscribers on the lowest layer. +Textured frames give every layer a realistic bitrate and steady packet flow. The source is 640x480 for one or two layers and 1280x720 for three layers, because the SDKs only split a simulcast source in three layers from 960 px wide. The SDKs derive the simulcast layers from the source: 480x360 + 640x480 from the 640x480 source (4:3 presets) and 320x180 + 640x360 + 1280x720 from the 1280x720 one (16:9 presets). +They publish with degradation preference `MAINTAIN_RESOLUTION`. The SDKs' +default for camera tracks is `MAINTAIN_FRAMERATE`, which lets libwebrtc scale +the source down under CPU overuse or bitrate pressure (1280x720 becomes +960x540, and every layer shrinks with it): the Java test identifies each layer +by its declared frame width, so on a loaded machine (GitHub-hosted runners) the +expected widths were never observed. With `MAINTAIN_RESOLUTION` libwebrtc drops +frames instead and the layer sizes stay as declared. They also capture at 15 fps +with several layers, because three software encoders at 30 fps trigger that +adaptation even faster. + ### Go SDK The Go SDK forwards pre-encoded samples and has no encoder. The publisher loops diff --git a/openvidu-test-e2e/src/test/resources/dotnet-publisher/Program.cs b/openvidu-test-e2e/src/test/resources/dotnet-publisher/Program.cs index 43fee9976..9554b75ac 100644 --- a/openvidu-test-e2e/src/test/resources/dotnet-publisher/Program.cs +++ b/openvidu-test-e2e/src/test/resources/dotnet-publisher/Program.cs @@ -21,6 +21,8 @@ var options = new TrackPublishOptions { VideoCodec = Enum.Parse(codec, ignoreCase: true), Source = Proto.TrackSource.SourceCamera, + // Keep the layer sizes under CPU load (see ../README.md) + DegradationPreference = Proto.DegradationPreference.MaintainResolution, }; if (codec == "vp8" || codec == "h264") { @@ -36,16 +38,21 @@ await room.LocalParticipant!.PublishTrackAsync(videoTrack, options); Console.WriteLine("TRACK_PUBLISHED"); var data = new byte[width * height * 4]; -byte n = 0; +int n = 0; while (true) { - n += 7; - for (int i = 0; i < data.Length; i += 4) + n++; + // Textured, moving frames (see ../README.md) + for (int y = 0; y < height; y++) { - data[i] = n; - data[i + 1] = (byte)(255 - n); - data[i + 2] = (byte)(n * 3); - data[i + 3] = 255; + for (int x = 0; x < width; x++) + { + int i = (y * width + x) * 4; + data[i] = (byte)(x + 7 * n); + data[i + 1] = (byte)(y + 3 * n); + data[i + 2] = (byte)(x + y + 11 * n); + data[i + 3] = 255; + } } videoSource.CaptureFrame(new VideoFrame(width, height, Proto.VideoBufferType.Rgba, data)); // 15 fps with several layers: several software encoders at 30 fps trigger CPU adaptation diff --git a/openvidu-test-e2e/src/test/resources/node-publisher/main.mjs b/openvidu-test-e2e/src/test/resources/node-publisher/main.mjs index fbbef28cf..ac54b1188 100644 --- a/openvidu-test-e2e/src/test/resources/node-publisher/main.mjs +++ b/openvidu-test-e2e/src/test/resources/node-publisher/main.mjs @@ -27,6 +27,9 @@ const track = LocalVideoTrack.createVideoTrack('node-video', source); const options = new TrackPublishOptions({ videoCodec: VideoCodec[codec.toUpperCase()], source: TrackSource.SOURCE_CAMERA, + // DegradationPreference.MAINTAIN_RESOLUTION (enum of @livekit/rtc-ffi-bindings, + // not re-exported by @livekit/rtc-node): keep the layer sizes under CPU load + degradationPreference: 2, }); if (codec === 'vp8' || codec === 'h264') { options.simulcast = layers > 1; diff --git a/openvidu-test-e2e/src/test/resources/python-publisher/main.py b/openvidu-test-e2e/src/test/resources/python-publisher/main.py index 225e814ca..2799c2d66 100644 --- a/openvidu-test-e2e/src/test/resources/python-publisher/main.py +++ b/openvidu-test-e2e/src/test/resources/python-publisher/main.py @@ -8,6 +8,25 @@ from livekit import rtc LAYERS = int(os.environ.get("VIDEO_LAYERS", "1")) # The SDK only splits a simulcast source in three layers from 960 px wide WIDTH, HEIGHT = (1280, 720) if LAYERS == 3 else (640, 480) +# Textured, moving RGBA frames (see ../README.md): a ramp long enough to slice +# any row of any frame out of it at C speed +RAMP = bytes(i & 0xFF for i in range(WIDTH + HEIGHT + 256)) +ALPHA = b"\xff" * WIDTH + + +def textured_frame(n): + """RGBA frame n of the animation: R = x + 7n, G = y + 3n, B = x + y + 11n.""" + frame = bytearray(WIDTH * HEIGHT * 4) + red_start = (7 * n) & 0xFF + red = RAMP[red_start:red_start + WIDTH] + for y in range(HEIGHT): + row = memoryview(frame)[y * WIDTH * 4:(y + 1) * WIDTH * 4] + blue_start = (y + 11 * n) & 0xFF + row[0::4] = red + row[1::4] = bytes([(y + 3 * n) & 0xFF]) * WIDTH + row[2::4] = RAMP[blue_start:blue_start + WIDTH] + row[3::4] = ALPHA + return bytes(frame) async def main(): codec = os.environ["VIDEO_CODEC"] @@ -24,6 +43,8 @@ async def main(): options = rtc.TrackPublishOptions( video_codec=getattr(rtc.VideoCodec, codec.upper()), source=rtc.TrackSource.SOURCE_CAMERA, + # Keep the layer sizes under CPU load (see ../README.md) + degradation_preference=rtc.DegradationPreference.MAINTAIN_RESOLUTION, ) if codec in ("vp8", "h264"): options.simulcast = LAYERS > 1 @@ -34,11 +55,12 @@ async def main(): # The Java test waits for this exact log line before asserting print("TRACK_PUBLISHED", flush=True) + # A short loop of pre-rendered frames keeps the capture loop cheap + frames = [textured_frame(n) for n in range(16)] n = 0 while True: - n = (n + 7) % 256 - data = bytes((n, 255 - n, (n * 3) % 256, 255)) * (WIDTH * HEIGHT) - source.capture_frame(rtc.VideoFrame(WIDTH, HEIGHT, rtc.VideoBufferType.RGBA, data)) + n += 1 + source.capture_frame(rtc.VideoFrame(WIDTH, HEIGHT, rtc.VideoBufferType.RGBA, frames[n % len(frames)])) # 15 fps with several layers: several software encoders at 30 fps trigger CPU adaptation await asyncio.sleep(1 / 15 if LAYERS > 1 else 1 / 30) diff --git a/openvidu-test-e2e/src/test/resources/rust-publisher/src/main.rs b/openvidu-test-e2e/src/test/resources/rust-publisher/src/main.rs index 8e5ace91c..3fcef4795 100644 --- a/openvidu-test-e2e/src/test/resources/rust-publisher/src/main.rs +++ b/openvidu-test-e2e/src/test/resources/rust-publisher/src/main.rs @@ -2,7 +2,7 @@ // See ../README.md use std::{env, time::Duration}; -use livekit::options::{TrackPublishOptions, VideoCodec}; +use livekit::options::{DegradationPreference, TrackPublishOptions, VideoCodec}; use livekit::track::{LocalTrack, LocalVideoTrack, TrackSource}; use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation}; use livekit::webrtc::video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}; @@ -17,6 +17,8 @@ async fn main() { let mut options = TrackPublishOptions { source: TrackSource::Camera, + // Keep the layer sizes under CPU load (see ../README.md) + degradation_preference: Some(DegradationPreference::MaintainResolution), video_codec: match codec.as_str() { "vp8" => VideoCodec::VP8, "h264" => VideoCodec::H264, @@ -62,13 +64,24 @@ async fn main() { frame_metadata: None, buffer: I420Buffer::new(width, height), }; - let mut n: u8 = 0; + let (stride_y, stride_u, stride_v) = frame.buffer.strides(); + let mut n: u32 = 0; loop { - n = n.wrapping_add(7); + n += 1; + // Textured, moving frames (see ../README.md): a diagonal ramp on the luma + // plane and slower ramps on chroma let (data_y, data_u, data_v) = frame.buffer.data_mut(); - data_y.fill(n); - data_u.fill(255 - n); - data_v.fill(n.wrapping_mul(3)); + for y in 0..height as usize { + for x in 0..width as usize { + data_y[y * stride_y as usize + x] = (x + y + 7 * n as usize) as u8; + } + } + for y in 0..(height as usize + 1) / 2 { + for x in 0..(width as usize + 1) / 2 { + data_u[y * stride_u as usize + x] = (x + 3 * n as usize) as u8; + data_v[y * stride_v as usize + x] = (y + 5 * n as usize) as u8; + } + } source.capture_frame(&frame); // 15 fps with several layers: several software encoders at 30 fps trigger CPU adaptation tokio::time::sleep(Duration::from_millis(if layers > 1 { 66 } else { 33 })).await;