e2e: verify every mat-select quality change and diagnose stalled subscriber videos

openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java

@@ -9,7 +9,6 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -260,6 +259,45 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e {
 		return isStatPresent(statValue) ? String.valueOf(statValue) : "absent";
 	}
 
+	// Every subscriber stat that says something about the state of the inbound
+	// video, so that a wait that times out reports what it actually saw instead
+	// of only the one value it was comparing
+	protected String describeSubscriberVideoLayer(JsonObject layer) {
+		return "Last observed: frameWidth=" + describeStat(getLayerCounter(layer, "frameWidth")) + " frameHeight="
+				+ describeStat(getLayerCounter(layer, "frameHeight")) + " framesPerSecond="
+				+ describeStat(getLayerCounter(layer, "framesPerSecond")) + " framesReceived="
+				+ describeStat(getLayerCounter(layer, "framesReceived")) + " framesDecoded="
+				+ describeStat(getLayerCounter(layer, "framesDecoded")) + " keyFramesDecoded="
+				+ describeStat(getLayerCounter(layer, "keyFramesDecoded")) + " framesDropped="
+				+ describeStat(getLayerCounter(layer, "framesDropped")) + " freezeCount="
+				+ describeStat(getLayerCounter(layer, "freezeCount")) + " bytesReceived="
+				+ describeStat(getLayerCounter(layer, "bytesReceived"));
+	}
+
+	// The three states a stalled subscriber video can be in, told apart by
+	// framesReceived (frames the depacketizer assembled, before the decoder) and
+	// framesDecoded. bytesReceived alone cannot tell them apart: it also counts
+	// retransmissions and the padding the SFU sends to probe for bandwidth, so it
+	// grows even while no frame at all reaches the decoder
+	protected String diagnoseSubscriberVideoLayer(JsonObject layer) {
+		long framesReceived = getLayerCounter(layer, "framesReceived");
+		long framesDecoded = getLayerCounter(layer, "framesDecoded");
+		if (!isStatPresent(framesReceived) || !isStatPresent(framesDecoded)) {
+			return "getStats() reported no frame counters for this track, so the subscriber never got as far as"
+					+ " receiving media on it";
+		}
+		if (framesReceived <= 0) {
+			return "The subscriber is not receiving assembled frames at all: the media is not reaching it";
+		}
+		if (framesDecoded <= 0) {
+			return "The subscriber IS receiving assembled frames (" + framesReceived
+					+ ") but decoded none of them: the media that reaches it is undecodable (a Producer bound to"
+					+ " the wrong codec, or a missing or wrong dependency descriptor)";
+		}
+		return "The subscriber received " + framesReceived + " assembled frame(s) and decoded " + framesDecoded
+				+ " of them, so media did flow at some point";
+	}
+
 	// If rid is null, retrieve the first layer
 	protected JsonElement getPublisherVideoLayerAttribute(OpenViduTestappUser user, WebElement publisherVideo,
 			String rid,
@@ -296,25 +334,31 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e {
 	}
 
 	protected void waitUntilSubscriberFramesPerSecondNotZero(OpenViduTestappUser user, WebElement videoElement) {
-		// Kept across iterations only to tell "absent" from "present and 0" if the
+		// Kept across iterations only to report what the last sample actually held
-		// wait times out. Those two mean very different things here, and the
+		// if the wait times out. An absent framesPerSecond and a present 0 mean very
-		// original message ("waiting for framesPerSecond to exist") named neither
+		// different things here, and neither of them says whether the media reached
-		final AtomicLong lastFps = new AtomicLong(-1);
+		// the subscriber at all: only framesReceived (frames the depacketizer
+		// assembled) against framesDecoded tells "nothing is arriving" from
+		// "something is arriving that the decoder cannot use"
+		final JsonObject[] lastLayer = { new JsonObject() };
 		this.waitUntilAux(user, videoElement, () -> {
 			// Chrome only starts reporting framesPerSecond once the decoder has
 			// produced frames for a whole second, so right after playback starts
 			// it is legitimately absent for a while: a "not yet", not a failure
 			JsonObject layer = this.getSubscriberVideoLayer(user, videoElement);
+			lastLayer[0] = layer;
 			long fps = this.getLayerCounter(layer, "framesPerSecond");
-			lastFps.set(fps);
 			return isStatPresent(fps) && fps > 0;
 		}, () -> {
-			long fps = lastFps.get();
+			long fps = this.getLayerCounter(lastLayer[0], "framesPerSecond");
 			return "Timeout waiting for video track to have a framesPerSecond greater than 0. Last value: "
-					+ describeStat(fps) + (isStatPresent(fps) ? ""
+					+ describeStat(fps)
+					+ (isStatPresent(fps) ? ""
 							: ". Chrome omits framesPerSecond altogether when the decoder produced no frame"
 									+ " during the last second, so this subscriber video was frozen for the"
-									+ " whole wait, not merely slow to start");
+									+ " whole wait, not merely slow to start")
+					+ ". " + describeSubscriberVideoLayer(lastLayer[0]) + ". "
+					+ diagnoseSubscriberVideoLayer(lastLayer[0]);
 		});
 	}
 
@@ -336,13 +380,8 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e {
 			lastLayer[0] = layer;
 			long frameWidth = this.getLayerCounter(layer, "frameWidth");
 			return isStatPresent(frameWidth) && frameWidth == expectedFrameWidth;
-		}, () -> "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth
+		}, () -> "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth + ". "
-				+ ". Last observed: frameWidth=" + describeStat(getLayerCounter(lastLayer[0], "frameWidth"))
+				+ describeSubscriberVideoLayer(lastLayer[0]));
-				+ " frameHeight=" + describeStat(getLayerCounter(lastLayer[0], "frameHeight")) + " framesPerSecond="
-				+ describeStat(getLayerCounter(lastLayer[0], "framesPerSecond")) + " framesDecoded="
-				+ describeStat(getLayerCounter(lastLayer[0], "framesDecoded")) + " keyFramesDecoded="
-				+ describeStat(getLayerCounter(lastLayer[0], "keyFramesDecoded")) + " bytesReceived="
-				+ describeStat(getLayerCounter(lastLayer[0], "bytesReceived")));
 	}
 
 	protected void waitUntilSubscriberFrameHeightIs(OpenViduTestappUser user, WebElement videoElement,
@@ -356,11 +395,18 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e {
 
 	protected void waitUntilSubscriberFrameWidthChanges(OpenViduTestappUser user, WebElement videoElement,
 			final int oldFrameWidth, final boolean shouldBeHigher) {
+		final JsonObject[] lastLayer = { new JsonObject() };
 		this.waitUntilAux(user, videoElement, () -> {
 			JsonObject layer = this.getSubscriberVideoLayer(user, videoElement);
+			lastLayer[0] = layer;
 			long frameWidth = this.getLayerCounter(layer, "frameWidth");
 			return isStatPresent(frameWidth) && frameWidth != oldFrameWidth;
-		}, "Timeout waiting for video track to reach a " + (shouldBeHigher ? "higher" : "lower") + " resolution");
+		}, () -> "Timeout waiting for video track to reach a " + (shouldBeHigher ? "higher" : "lower")
+				+ " resolution than " + oldFrameWidth + ". " + describeSubscriberVideoLayer(lastLayer[0]) + ". "
+				// frameWidth is the width of the last frame the decoder produced, so a
+				// video that froze keeps reporting the old width forever and looks
+				// exactly like a layer switch that never happened
+				+ diagnoseSubscriberVideoLayer(lastLayer[0]));
 		int newFrameWidth = this.getSubscriberVideoFrameWidth(user, videoElement);
 		if (shouldBeHigher) {
 			Assertions.assertTrue(newFrameWidth > oldFrameWidth,
@@ -744,6 +790,27 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e {
 				+ maxAttempts + " attempts");
 	}
 
+	/**
+	 * Selects the max video quality (LOW, MEDIUM or HIGH) of the remote video of a
+	 * testapp instance, closing the track info dialog first if it is open (its
+	 * backdrop covers the video controls).
+	 *
+	 * The selection is verified (selectMatOption). An unverified click that only
+	 * opens and closes the mat-select panel leaves the quality untouched and
+	 * raises nothing: setVideoQuality() is never called, no UpdateTrackSettings
+	 * ever reaches the server, and the wait that follows times out reporting that
+	 * the subscriber never changed layer, when in fact nothing was ever asked of
+	 * it.
+	 */
+	protected void selectSubscriberVideoQuality(OpenViduTestappUser user, String instanceSelector, String quality)
+			throws InterruptedException {
+		if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) {
+			this.waitAndClick(user, "#close-dialog-btn");
+			Thread.sleep(300);
+		}
+		this.selectMatOption(user, instanceSelector + " #max-video-quality", quality);
+	}
+
 	/**
 	 * Waits until the element matching the CSS selector is present, displayed and
 	 * enabled, then clicks it, retrying every 250 ms for up to 10 seconds on the

openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java

@@ -268,12 +268,7 @@ public class OpenViduTestAppE2eServerSdkTest extends AbstractOpenViduTestappE2eT
 	 * over the page, which would block the later clicks on the video controls.
 	 */
 	private void selectSubscriberVideoQuality(Subscriber subscriber, String quality) throws InterruptedException {
-		OpenViduTestappUser user = subscriber.user();
+		this.selectSubscriberVideoQuality(subscriber.user(), subscriber.instanceSelector(), quality);
-		if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) {
-			user.getDriver().findElement(By.cssSelector("#close-dialog-btn")).click();
-			Thread.sleep(300);
-		}
-		this.selectMatOption(user, subscriber.instanceSelector() + " #max-video-quality", quality);
 	}
 
 	/**

openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java

@@ -2640,12 +2640,10 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 		user.getEventManager().waitUntilEventReaches(2, "trackSubscribed", "RoomEvent", 1);
 
 		// Manually change video quality of first subscriber to q
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 
 		// Manually change video quality of second subscriber to f
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-2 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-2", "HIGH");
-		this.waitAndClick(user, "mat-option.mode-HIGH");
 
 		subscriberVideo1 = user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 video.remote"));
 		WebElement subscriberVideo2 = user.getDriver().findElement(By.cssSelector("#openvidu-instance-2 video.remote"));
@@ -2672,8 +2670,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 		secondSubscriberToggle.click();
 
 		// Manually change video quality of second subscriber to h
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-2 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-2", "MEDIUM");
-		this.waitAndClick(user, "mat-option.mode-MEDIUM");
 
 		this.waitUntilPublisherLayerActive(user, publisherVideo, "q", true);
 		this.waitUntilPublisherLayerActive(user, publisherVideo, "h", true);
@@ -2836,18 +2833,15 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 		this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo);
 
 		// Manually change video quality of subscriber to h
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", "MEDIUM");
-		this.waitAndClick(user, "mat-option.mode-MEDIUM");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, h);
 
 		// Manually change video quality of subscriber to q
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, q);
 
 		// Manually change video quality of subscriber to f
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", "HIGH");
-		this.waitAndClick(user, "mat-option.mode-HIGH");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, f);
 
 		gracefullyLeaveParticipants(user, 2);
@@ -3426,8 +3420,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 
 		// Even after forcing the low quality layer in the subscriber, with dynacast
 		// enabled, the entire SVC stream should remain active in publisher
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 960);
 		this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo);
 
@@ -3646,8 +3639,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 		// layers
 		// at that size, so it stops producing the top one
 		final long switchStart = System.currentTimeMillis();
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #restart-video-resolution")).click();
+		this.selectMatOption(user, "#openvidu-instance-0 #restart-video-resolution", "640x360");
-		this.waitAndClick(user, "mat-option.res-640x360");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 640);
 		final long switchMillis = System.currentTimeMillis() - switchStart;
 		this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo);
@@ -3666,8 +3658,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 						+ " ms)");
 
 		// And it must be able to follow the publisher back up (this needs a keyframe)
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #restart-video-resolution")).click();
+		this.selectMatOption(user, "#openvidu-instance-0 #restart-video-resolution", "1920x1080");
-		this.waitAndClick(user, "mat-option.res-1920x1080");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920);
 		this.waitUntilSubscriberFramesDecodedIncrease(user, subscriberVideo);
 
@@ -3910,7 +3901,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 	// sufficient for that size. Only "LOW" (small element) and "HIGH" (large
 	// element) are meaningful in that case.
 	private void switchSubscriberSpatialLayer(OpenViduTestappUser user, WebElement subscriberVideo,
-			boolean adaptiveStream, String quality) {
+			boolean adaptiveStream, String quality) throws InterruptedException {
 		if (adaptiveStream) {
 			if ("LOW".equals(quality)) {
 				changeElementSize(user, subscriberVideo, 80, 100);
@@ -3918,8 +3909,7 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 				changeElementSize(user, subscriberVideo, 1000, 700);
 			}
 		} else {
-			user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 #max-video-quality")).click();
+			this.selectSubscriberVideoQuality(user, "#openvidu-instance-1", quality);
-			this.waitAndClick(user, "mat-option.mode-" + quality);
 		}
 	}
 
@@ -4819,38 +4809,30 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
 	private void testThreeLayers(OpenViduTestappUser user, WebElement subscriberVideo) throws InterruptedException {
 		// Check manual simulcast changes
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 640);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "MEDIUM");
-		this.waitAndClick(user, "mat-option.mode-MEDIUM");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1280);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "HIGH");
-		this.waitAndClick(user, "mat-option.mode-HIGH");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920);
 	}
 
 	private void testTwoLayers(OpenViduTestappUser user, WebElement subscriberVideo) throws InterruptedException {
 		// Check manual simulcast changes
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 960);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 480);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "MEDIUM");
-		this.waitAndClick(user, "mat-option.mode-MEDIUM");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 960);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 480);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "HIGH");
-		this.waitAndClick(user, "mat-option.mode-HIGH");
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 960);
 	}
 
 	private void testNoSimulcast(OpenViduTestappUser user, WebElement subscriberVideo) throws InterruptedException {
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920);
-		user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click();
+		this.selectSubscriberVideoQuality(user, "#openvidu-instance-0", "LOW");
-		this.waitAndClick(user, "mat-option.mode-LOW");
 		// Without simulcast video should remain in high quality
 		Thread.sleep(4000);
 		this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920);

openvidu-testapp/src/app/components/video-track/video-track.component.html

@@ -23,7 +23,7 @@
       <mat-form-field id="restart-video-resolution" class="video-btn quality-option" matTooltip="Restart video at resolution" matTooltipClass="custom-tooltip">
         <mat-select [(value)]="restartResolution" (selectionChange)="onRestartResolutionChange()">
           @for (r of ['1920x1080', '1280x720', '640x360', '320x180']; track r) {
-            <mat-option [value]="r" [ngClass]="'res-' + r">{{r}}</mat-option>
+            <mat-option [value]="r" [id]="'mat-option-' + r" [ngClass]="'res-' + r">{{r}}</mat-option>
           }
         </mat-select>
       </mat-form-field>