diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java index 5bc4e30ab..07804993c 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java @@ -790,6 +790,33 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e { } } + // The local camera track of an instance: the video element id carries the + // track source, and a track keeps its component while published + protected static final String LOCAL_CAMERA_TRACK = "app-video-track:has(video.local[id*='--camera--'])"; + + /** + * Publishes a new camera track from the given instance ("Add video" button) and + * waits until that instance has published expectedLocalTrackPublished local + * tracks in total. + */ + protected void publishCamera(OpenViduTestappUser user, int instance, int expectedLocalTrackPublished) + throws Exception { + this.waitAndClick(user, "#openvidu-instance-" + instance + " .add-video-btn"); + user.getEventManager().waitUntilEventReaches(instance, "localTrackPublished", "RoomEvent", + expectedLocalTrackPublished); + } + + /** + * Unpublishes the camera track of the given instance and waits until that + * instance has unpublished expectedLocalTrackUnpublished local tracks in total. + */ + protected void unpublishCamera(OpenViduTestappUser user, int instance, int expectedLocalTrackUnpublished) + throws Exception { + this.waitAndClick(user, "#openvidu-instance-" + instance + " " + LOCAL_CAMERA_TRACK + " .publish-unpublish-video"); + user.getEventManager().waitUntilEventReaches(instance, "localTrackUnpublished", "RoomEvent", + expectedLocalTrackUnpublished); + } + protected void createIngress(OpenViduTestappUser user, String preset, String codec, boolean simulcast, String urlType, String urlUri) throws InterruptedException { diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eEgressProTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eEgressProTest.java index 612976b71..2dd1893b5 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eEgressProTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eEgressProTest.java @@ -3165,8 +3165,7 @@ public class OpenViduTestAppE2eEgressProTest extends AbstractOpenViduTestappE2eT if (cameras.isEmpty()) { cameras.addAll(before); } - click(".add-video-btn"); - waitForEvent("localTrackPublished", ++published); + publishCamera(user, instance, ++published); cameras.add(newId("video", "camera", before)); parkMouse(); } @@ -3284,8 +3283,7 @@ public class OpenViduTestAppE2eEgressProTest extends AbstractOpenViduTestappE2eT } void unpublishVideo() throws Exception { - click(CAMERA + " .publish-unpublish-video"); - waitForEvent("localTrackUnpublished", ++unpublished); + unpublishCamera(user, instance, ++unpublished); } void unpublishAudio() throws Exception { diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java index c86317fa9..6711082a0 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java @@ -3220,6 +3220,184 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { gracefullyLeaveParticipants(user, 2); } + // --- Subscriber bandwidth estimate across video pauses ------------------------- + // + // A subscriber connection that carries only audio must not be probed for + // bandwidth (nothing can host the probes), and its estimate must survive the + // pause so that the video comes back on its top layer. The three tests below + // drive the ways a connection gets there: the publisher mutes the camera, the + // subscriber disables the video, and the camera is unpublished + + // The top simulcast layer of the fake camera + private static final int CAMERA_TOP_LAYER_WIDTH = 640; + // Several periods of the server's periodic bandwidth probing (every 5 s while + // the connection sends much less than its estimate) + private static final long VIDEO_PAUSE_MILLIS = 20000; + + /** + * Publisher (instance 0) with audio and simulcast video and no dynacast, and a + * subscriber (instance 1) without adaptive stream, so that it keeps the video + * subscribed while it is hidden. Returns the subscriber's video once its top + * layer flows and the server's estimate has had time to settle. + */ + private WebElement connectPublisherAndSubscriberForBwe(OpenViduTestappUser user) throws Exception { + this.addPublisher(user, false, true, false, false, true, true, null, null, null); + this.addSubscriber(user, false); + + user.getDriver().findElements(By.className("connect-btn")).forEach(el -> el.sendKeys(Keys.ENTER)); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 2); + + WebElement subscriberVideo = user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 video.remote")); + this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo); + this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, CAMERA_TOP_LAYER_WIDTH); + Thread.sleep(5000); + return subscriberVideo; + } + + /** + * With the video just paused: checks that no frame is decoded and that only the + * audio reaches the subscriber during VIDEO_PAUSE_MILLIS. Returns the frames + * decoded so far, for the resume check. + */ + private long assertOnlyAudioWhilePaused(OpenViduTestappUser user, WebElement subscriberVideo) throws Exception { + // Let the video packets in flight arrive before taking the reference counters + Thread.sleep(1000); + + JsonObject pauseStart = this.getSubscriberVideoLayer(user, subscriberVideo); + final long framesDecodedPaused = this.getLayerCounter(pauseStart, "framesDecoded"); + final long bytesAtPauseStart = this.getLayerCounter(pauseStart, "transportBytesReceived"); + Assertions.assertTrue(isStatPresent(framesDecodedPaused), "framesDecoded is not reported"); + Assertions.assertTrue(isStatPresent(bytesAtPauseStart), "transportBytesReceived is not reported"); + + Thread.sleep(VIDEO_PAUSE_MILLIS); + + JsonObject pauseEnd = this.getSubscriberVideoLayer(user, subscriberVideo); + final long bytesWhilePaused = this.getLayerCounter(pauseEnd, "transportBytesReceived") - bytesAtPauseStart; + Assertions.assertEquals(framesDecodedPaused, this.getLayerCounter(pauseEnd, "framesDecoded"), + "Frames were decoded while the video was paused"); + // Only the audio and the RTCP may have reached the subscriber, well under + // 100 kbps. A server that probes a connection with no video to carry the + // probes shows up here as hundreds of KB of RTX padding per probing period + final long maxBytesWhilePaused = 100 * 1000 / 8 * VIDEO_PAUSE_MILLIS / 1000; + Assertions.assertTrue(bytesWhilePaused < maxBytesWhilePaused, + "The subscriber received " + bytesWhilePaused + " bytes during the " + (VIDEO_PAUSE_MILLIS / 1000) + + " s video pause, more than the " + maxBytesWhilePaused + + " bytes the audio accounts for: the server is probing the connection while it has no video to send"); + return framesDecodedPaused; + } + + /** + * With the video just resumed: the first frames to arrive must already be the + * top layer. The server kept its bandwidth estimate across the pause, so it picks + * that layer at once. An estimate lost during the pause brings the video back on + * its lowest layer and climbs for a couple of seconds. + */ + private void assertVideoResumesOnTopLayer(OpenViduTestappUser user, WebElement subscriberVideo, + long framesDecodedBefore) throws Exception { + final JsonObject[] firstFrames = { new JsonObject() }; + this.waitUntilAux(user, subscriberVideo, () -> { + JsonObject layer = this.getSubscriberVideoLayer(user, subscriberVideo); + if (this.getLayerCounter(layer, "framesDecoded") > framesDecodedBefore) { + firstFrames[0] = layer; + return true; + } + return false; + }, "Timeout waiting for the video to resume"); + Assertions.assertEquals(CAMERA_TOP_LAYER_WIDTH, this.getLayerCounter(firstFrames[0], "frameWidth"), + "The video came back below its top layer: the server lost its bandwidth estimate while the video was paused. " + + describeSubscriberVideoLayer(firstFrames[0])); + + // And it stays there + this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo); + Assertions.assertEquals(CAMERA_TOP_LAYER_WIDTH, this.getSubscriberVideoFrameWidth(user, subscriberVideo), + "The video left its top layer after resuming"); + } + + @Test + @DisplayName("Camera mute keeps the subscriber bandwidth estimate") + void cameraMuteKeepsSubscriberBweTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("Camera mute keeps the subscriber bandwidth estimate"); + + WebElement subscriberVideo = connectPublisherAndSubscriberForBwe(user); + + // The publisher mutes the camera. Its audio keeps flowing + user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 .mute-unmute-video")).sendKeys(Keys.ENTER); + user.getEventManager().waitUntilEventReaches("trackMuted", "RoomEvent", 1); + + long framesDecodedPaused = assertOnlyAudioWhilePaused(user, subscriberVideo); + + // The publisher unmutes + user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 .mute-unmute-video")).sendKeys(Keys.ENTER); + user.getEventManager().waitUntilEventReaches("trackUnmuted", "RoomEvent", 1); + + assertVideoResumesOnTopLayer(user, subscriberVideo, framesDecodedPaused); + + gracefullyLeaveParticipants(user, 2); + } + + @Test + @DisplayName("Subscriber disabled video keeps the subscriber bandwidth estimate") + void subscriberDisabledVideoKeepsSubscriberBweTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("Subscriber disabled video keeps the subscriber bandwidth estimate"); + + WebElement subscriberVideo = connectPublisherAndSubscriberForBwe(user); + + // The subscriber disables the video (what adaptive stream does with a hidden + // video). The server stops forwarding it after a 100 ms debounce + user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 .toggle-video-enabled")) + .sendKeys(Keys.ENTER); + + long framesDecodedPaused = assertOnlyAudioWhilePaused(user, subscriberVideo); + + // The subscriber enables the video again + user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 .toggle-video-enabled")) + .sendKeys(Keys.ENTER); + + assertVideoResumesOnTopLayer(user, subscriberVideo, framesDecodedPaused); + + gracefullyLeaveParticipants(user, 2); + } + + @Test + @OnlyMediasoup + @DisplayName("Camera unpublish keeps the subscriber bandwidth estimate") + void cameraUnpublishKeepsSubscriberBweTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("Camera unpublish keeps the subscriber bandwidth estimate"); + + WebElement subscriberVideo = connectPublisherAndSubscriberForBwe(user); + + // The publisher unpublishes the camera: the subscriber's connection is left + // with the audio alone and no video track at all, for several periods of the + // server's periodic bandwidth probing. The probes it would send in this state + // have no track to be routed to, so the subscriber drops them unaccounted and + // they cannot be measured from here: what shows is the estimate they ruin + this.unpublishCamera(user, 0, 1); + user.getEventManager().waitUntilEventReaches("trackUnpublished", "RoomEvent", 1); + Thread.sleep(VIDEO_PAUSE_MILLIS); + + // The publisher publishes the camera again: a new track, subscribed anew. With + // mediasoup its first frames are the top layer, since the connection's + // estimate was kept. The pion engine starts a new track on its lowest layer + // by design, hence the engine restriction of this test + this.publishCamera(user, 0, 3); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 3); + subscriberVideo = user.getWaiter() + .until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("#openvidu-instance-1 video.remote"))); + + assertVideoResumesOnTopLayer(user, subscriberVideo, 0); + + gracefullyLeaveParticipants(user, 2); + } + @Test @DisplayName("Enable disable track with dynacast") void enableDisableTrackWithDynacastTest() throws Exception { diff --git a/openvidu-testapp/src/app/components/video-track/video-track.component.ts b/openvidu-testapp/src/app/components/video-track/video-track.component.ts index a5448aefc..10ed927ce 100644 --- a/openvidu-testapp/src/app/components/video-track/video-track.component.ts +++ b/openvidu-testapp/src/app/components/video-track/video-track.component.ts @@ -114,11 +114,18 @@ export class VideoTrackComponent extends TrackComponent { const videoLayers: any[] = []; let stats = await (this._track! as VideoTrack).getRTCStatsReport(); let codecs = new Map(); + let transport: RTCTransportStats | undefined; stats?.forEach((report) => { if (report.type === 'codec') { // Store for matching with codecId in 'outbound-rtp' or 'inbound-rtp' reports codecs.set(report.id, report); } + if (report.type === 'transport') { + // Whole-connection counters: RTP, RTX padding and RTCP of every track of + // this peer connection, so a subscriber can see traffic that its own + // 'inbound-rtp' does not count + transport = report as RTCTransportStats; + } if (report.type === 'outbound-rtp') { const reportTyped = report as RTCOutboundRtpStreamStats; videoLayers.push({ @@ -161,6 +168,10 @@ export class VideoTrackComponent extends TrackComponent { if (codecs.has(layer.codecId)) { layer.codec = codecs.get(layer.codecId).mimeType; } + if (transport) { + layer.transportBytesReceived = transport.bytesReceived; + layer.transportPacketsReceived = transport.packetsReceived; + } }); return JSON.stringify(videoLayers, null, 2); };