diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java index 2c17633f5..b8f046315 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java @@ -3,14 +3,25 @@ package io.openvidu.test.e2e; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; +import java.util.concurrent.Callable; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.openqa.selenium.By; import org.openqa.selenium.Keys; +import org.openqa.selenium.TakesScreenshot; import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.ui.ExpectedConditions; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.openvidu.test.browsers.BrowserUser; +import static org.openqa.selenium.OutputType.BASE64; + public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e { protected Collection testappUsers = new HashSet<>(); @@ -66,4 +77,611 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e { super.dispose(); } + protected static final long WAIT_UNTIL_MAX_MILLIS = 20000; + + // Minimum average frame rate that a subscriber video must sustain over a + // window of at least MIN_FRAMES_DECODED_WINDOW_MILLIS to be considered + // properly decoded and played + protected static final long MIN_FRAMES_DECODED_FPS = 4; + protected static final long MIN_FRAMES_DECODED_WINDOW_MILLIS = 2000; + + protected static void pullRemoteBrowserImages() { + pullRemoteBrowserImage("REMOTE_URL_CHROME", "selenium/standalone-chrome:" + CHROME_VERSION); + pullRemoteBrowserImage("REMOTE_URL_FIREFOX", "selenium/standalone-firefox:" + FIREFOX_VERSION); + pullRemoteBrowserImage("REMOTE_URL_EDGE", "selenium/standalone-edge:" + EDGE_VERSION); + } + + protected static void pullRemoteBrowserImage(String remoteUrlProperty, String image) { + if (System.getProperty(remoteUrlProperty) == null) { + return; // This browser runs as a native driver here. No Docker image to pull + } + try { + log.info("Pre-pulling Selenium image {}", image); + commandLine.executeCommand("docker pull " + image, 300); + } catch (Exception e) { + System.err.println("Pre-pull of " + image + " failed: " + e.getMessage()); + } + } + + protected int countNumberOfPublishedLayers(OpenViduTestappUser user, WebElement publisherVideo) { + JsonArray json = this.getLayersAsJsonArray(user, publisherVideo); + return json.size(); + } + + protected int getSubscriberVideoFrameWidth(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "frameWidth", JsonElement::getAsInt); + } + + protected int getSubscriberVideoFrameHeight(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "frameHeight", JsonElement::getAsInt); + } + + protected long getSubscriberVideoBytesReceived(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "bytesReceived", JsonElement::getAsLong); + } + + protected int getSubscriberVideoFramesPerSecond(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "framesPerSecond", JsonElement::getAsInt); + } + + protected long getSubscriberVideoFramesDecoded(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "framesDecoded", JsonElement::getAsLong); + } + + protected long getSubscriberVideoFramesReceived(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "framesReceived", JsonElement::getAsLong); + } + + protected String getSubscriberVideoCodec(OpenViduTestappUser user, WebElement subscriberVideo) { + return getSubscriberVideoLayerStat(user, subscriberVideo, "codec", JsonElement::getAsString); + } + + protected T getSubscriberVideoLayerStat(OpenViduTestappUser user, WebElement subscriberVideo, String field, + java.util.function.Function extractor) { + final long deadline = System.currentTimeMillis() + WAIT_UNTIL_MAX_MILLIS; + JsonElement element = null; + do { + try { + element = getLayersAsJsonArray(user, subscriberVideo).get(0).getAsJsonObject().get(field); + } catch (Exception e) { + element = null; + } + if (element == null || element.isJsonNull()) { + try { + Thread.sleep(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } while ((element == null || element.isJsonNull()) && System.currentTimeMillis() < deadline); + if (element == null || element.isJsonNull()) { + Assertions.fail("Timeout waiting for " + field + " to exist"); + } + return extractor.apply(element); + } + + // Several stats of the same subscriber layer, taken from a single info dialog + // update. Sampling them one by one through getSubscriberVideoLayerStat would + // pay + // a full dialog read, and a stats refresh, for each one of them + protected JsonObject getSubscriberVideoLayer(OpenViduTestappUser user, WebElement subscriberVideo) { + JsonArray layers = this.getLayersAsJsonArray(user, subscriberVideo); + return layers.isEmpty() ? new JsonObject() : layers.get(0).getAsJsonObject(); + } + + // Cumulative counter of the given layer, or -1 if it is not there + protected long getLayerCounter(JsonObject layer, String field) { + JsonElement element = layer.get(field); + return element == null || element.isJsonNull() ? -1 : element.getAsLong(); + } + + // If rid is null, retrieve the first layer + protected JsonElement getPublisherVideoLayerAttribute(OpenViduTestappUser user, WebElement publisherVideo, + String rid, + String attribute) { + JsonArray json = this.getLayersAsJsonArray(user, publisherVideo); + JsonElement result; + if (rid != null) { + result = json.asList().stream().parallel() + .filter(jsonElement -> rid.equals(jsonElement.getAsJsonObject().get("rid").getAsString())).findAny() + .get(); + } else { + result = json.get(0); + } + return result.getAsJsonObject().get(attribute); + } + + protected String getLayersAsString(OpenViduTestappUser user, WebElement video) { + this.openInfoDialog(user, video); + user.getDriver().findElement(By.cssSelector("#update-value-btn")).click(); + WebElement textarea = user.getDriver().findElement(By.id("info-text-area")); + return textarea.getAttribute("value"); + } + + protected JsonArray getLayersAsJsonArray(OpenViduTestappUser user, WebElement video) { + String value = getLayersAsString(user, video); + return JsonParser.parseString(value).getAsJsonArray(); + } + + protected void waitUntilVideoLayersNotEmpty(OpenViduTestappUser user, WebElement videoElement) { + this.waitUntilAux(user, videoElement, () -> { + String value = getLayersAsString(user, videoElement); + return !value.isBlank() && !JsonParser.parseString(value).getAsJsonArray().isEmpty(); + }, "Timeout waiting video layers to not be empty"); + } + + protected void waitUntilSubscriberFramesPerSecondNotZero(OpenViduTestappUser user, WebElement videoElement) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoFramesPerSecond(user, videoElement) > 0; + }, "Timeout waiting for video track to have a framesPerSecond greater than 0"); + } + + protected void waitUntilSubscriberFramesPerSecondIs(OpenViduTestappUser user, WebElement videoElement, int fps) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoFramesPerSecond(user, videoElement) == fps; + }, "Timeout waiting for video track to have a framesPerSecond equal to " + fps); + } + + protected void waitUntilSubscriberFrameWidthIs(OpenViduTestappUser user, WebElement videoElement, + final int expectedFrameWidth) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoFrameWidth(user, videoElement) == expectedFrameWidth; + }, "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth); + } + + protected void waitUntilSubscriberFrameHeightIs(OpenViduTestappUser user, WebElement videoElement, + final int expectedFrameHeight) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoFrameHeight(user, videoElement) == expectedFrameHeight; + }, "Timeout waiting for video track to have a frameHeight of " + expectedFrameHeight); + } + + protected void waitUntilSubscriberFrameWidthChanges(OpenViduTestappUser user, WebElement videoElement, + final int oldFrameWidth, final boolean shouldBeHigher) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoFrameWidth(user, videoElement) != oldFrameWidth; + }, "Timeout waiting for video track to reach a " + (shouldBeHigher ? "higher" : "lower") + " resolution"); + int newFrameWidth = this.getSubscriberVideoFrameWidth(user, videoElement); + if (shouldBeHigher) { + Assertions.assertTrue(newFrameWidth > oldFrameWidth, + "Video track should have now a higher resolution, but it is not. Old width: " + oldFrameWidth + + ". New width: " + newFrameWidth); + } else { + Assertions.assertTrue(newFrameWidth < oldFrameWidth, + "Video track should have now a lower resolution, but it is not. Old width: " + oldFrameWidth + + ". New width: " + newFrameWidth); + } + } + + protected void waitUntilSubscriberBytesReceivedIncrease(OpenViduTestappUser user, WebElement videoElement, + final long previousBytesReceived) { + this.waitUntilAux(user, videoElement, () -> { + return this.getSubscriberVideoBytesReceived(user, videoElement) > previousBytesReceived; + }, "Timeout waiting for subscriber track to increase its bytesReceived from " + previousBytesReceived); + } + + // A subscriber video is only properly received AND played if its decoder keeps + // producing new frames at a sustained rate. Receiving bytes is not enough: a + // subscriber may receive media that it is not able to decode at all. And a + // single new decoded frame is not enough either: a video that only decodes one + // or two frames over a timespan of several seconds is a frozen video, not a + // playing one, and must fail the test. So framesDecoded is required to grow at + // MIN_FRAMES_DECODED_FPS or more, averaged over a window of at least + // MIN_FRAMES_DECODED_WINDOW_MILLIS + protected void waitUntilSubscriberFramesDecodedIncrease(OpenViduTestappUser user, WebElement videoElement) { + final long initialFramesDecoded = this.getSubscriberVideoFramesDecoded(user, videoElement); + final long initialFramesReceived = this.getSubscriberVideoFramesReceived(user, videoElement); + final long windowStart = System.currentTimeMillis(); + // Last sample taken by the loop, only to report it if the wait times out + final java.util.concurrent.atomic.AtomicLong lastFramesDecoded = new java.util.concurrent.atomic.AtomicLong(); + final java.util.concurrent.atomic.AtomicLong lastFramesReceived = new java.util.concurrent.atomic.AtomicLong(); + final java.util.concurrent.atomic.AtomicLong lastWindowMillis = new java.util.concurrent.atomic.AtomicLong(); + this.waitUntilAux(user, videoElement, () -> { + // Both counters must come from the very same dialog update: sampling + // them one by one would double the cost of every iteration + JsonObject layer = this.getSubscriberVideoLayer(user, videoElement); + long framesDecoded = this.getLayerCounter(layer, "framesDecoded"); + long framesReceived = this.getLayerCounter(layer, "framesReceived"); + if (framesDecoded < 0 || framesReceived < 0) { + return false; + } + long windowMillis = System.currentTimeMillis() - windowStart; + lastFramesDecoded.set(framesDecoded - initialFramesDecoded); + lastFramesReceived.set(framesReceived - initialFramesReceived); + lastWindowMillis.set(windowMillis); + // The window keeps growing while waiting, so a video that decodes a + // frame every now and then falls further behind the required rate + // instead of eventually satisfying it + return windowMillis >= MIN_FRAMES_DECODED_WINDOW_MILLIS + && lastFramesDecoded.get() * 1000 >= MIN_FRAMES_DECODED_FPS * windowMillis; + }, () -> { + long framesDecoded = lastFramesDecoded.get(); + long framesReceived = lastFramesReceived.get(); + long windowMillis = lastWindowMillis.get(); + // framesReceived counts the frames the depacketizer assembled, before + // handing them to the decoder. Comparing it against framesDecoded tells + // apart three failures that otherwise all look like "no video" + String diagnosis; + if (framesReceived <= 0) { + diagnosis = "The subscriber is not receiving assembled frames at all:" + + " the media is not reaching it"; + } else if (framesDecoded <= 0) { + diagnosis = "The subscriber IS receiving assembled frames (" + framesReceived + + ") but decoded none of them: the media that reaches it is undecodable" + + " (a Producer bound to the wrong codec, or a missing or wrong dependency" + + " descriptor)"; + } else { + diagnosis = "The subscriber received " + framesReceived + " assembled frame(s) and decoded " + + framesDecoded + " of them, but too slowly for a video that is actually playing"; + } + return "Timeout waiting for subscriber track to decode video at a sustained frame rate: only " + + framesDecoded + " frame(s) decoded in " + windowMillis + " ms (" + + String.format("%.2f", framesDecoded * 1000d / Math.max(1, windowMillis)) + + " fps), while at least " + MIN_FRAMES_DECODED_FPS + + " fps are required. Such a subscriber video is a frozen video. " + diagnosis; + }); + } + + protected void waitUntilPublisherBytesSentIncrease(OpenViduTestappUser user, WebElement videoElement, String rid, + final long previousBytesSent) { + this.waitUntilAux(user, videoElement, () -> { + return this.getPublisherVideoLayerAttribute(user, videoElement, rid, "bytesSent") + .getAsLong() > previousBytesSent; + }, "Timeout waiting for publisher track to increase its bytesSent from " + previousBytesSent); + } + + protected void waitUntilPublisherFramesEncodedIncrease(OpenViduTestappUser user, WebElement videoElement, + String rid, + final long previousFramesEncoded) { + this.waitUntilAux(user, videoElement, () -> { + return this.getPublisherVideoLayerAttribute(user, videoElement, rid, "framesEncoded") + .getAsLong() > previousFramesEncoded; + }, "Timeout waiting for publisher track to increase its framesEncoded from " + previousFramesEncoded); + } + + protected void waitUntilPublisherLayerActive(OpenViduTestappUser user, final WebElement publisherVideo, + final String rid, final boolean active) { + this.waitUntilAux(user, publisherVideo, () -> { + boolean currentlyActive = this.getPublisherVideoLayerAttribute(user, publisherVideo, rid, "active") + .getAsBoolean(); + if (active) { + JsonElement frameWidth = this.getPublisherVideoLayerAttribute(user, publisherVideo, rid, "frameWidth"); + return currentlyActive && frameWidth != null; + } else { + return !currentlyActive; + } + }, "Timeout waiting for video track layer to be " + (active ? "active" : "inactive")); + } + + protected void waitUntilAux(OpenViduTestappUser user, WebElement videoElement, + Callable breakFromLoopFunction, String errMsg) { + this.waitUntilAux(user, videoElement, breakFromLoopFunction, () -> errMsg); + } + + // Same as above, but building the error message only if the wait times out, so + // that it can report the values actually observed by the last iteration + protected void waitUntilAux(OpenViduTestappUser user, WebElement videoElement, + Callable breakFromLoopFunction, java.util.function.Supplier errMsg) { + try { + final long intervalWait = 250; + final long deadline = System.currentTimeMillis() + WAIT_UNTIL_MAX_MILLIS; + boolean breakFromLoop = false; + while (!breakFromLoop && System.currentTimeMillis() < deadline) { + try { + breakFromLoop = breakFromLoopFunction.call(); + } catch (Exception e1) { + e1.printStackTrace(); + } + if (breakFromLoop) { + break; + } else { + try { + Thread.sleep(intervalWait); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + if (!breakFromLoop) { + Assertions.fail(errMsg.get()); + } + } finally { + // Best-effort close of the info dialog + try { + if (!user.getDriver().findElements(By.cssSelector("#close-dialog-btn")).isEmpty()) { + this.waitForBackdropAndClick(user, "#close-dialog-btn"); + Thread.sleep(500); + } + } catch (Exception e) { + log.warn("Best-effort info-dialog close failed (ignored): {}", e.getMessage()); + } + } + } + + protected void openInfoDialog(OpenViduTestappUser user, WebElement video) { + String videoId = video.getDomProperty("id"); + // Open the track info dialog if required + boolean dialogWasOpened; + if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) { + // Dialog already opened + if (!user.getDriver().findElement(By.cssSelector("#subtitle")).getText().equals(videoId)) { + // Wrong dialog + this.waitForBackdropAndClick(user, "#close-dialog-btn"); + this.waitForBackdropAndClick(user, "#" + videoId + " ~ .bottom-div .video-track-info"); + dialogWasOpened = true; + } else { + dialogWasOpened = false; + } + } else { + // Dialog is not opened + this.waitForBackdropAndClick(user, "#" + videoId + " ~ .bottom-div .video-track-info"); + dialogWasOpened = true; + } + if (dialogWasOpened) { + try { + Thread.sleep(300); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + protected void addPublisherSubscriber(OpenViduTestappUser user, boolean hasAudio, boolean hasVideo) + throws InterruptedException { + this.addPublisher(user, true, true, true, true, hasAudio, hasVideo, null, null, null); + } + + protected void addOnlyPublisherVideo(OpenViduTestappUser user, boolean simulcast, boolean dynacast, boolean hd) + throws InterruptedException { + if (hd) { + this.addPublisher(user, false, simulcast, dynacast, false, false, true, 1920, 1080, null); + } else { + this.addPublisher(user, false, simulcast, dynacast, false, false, true, null, null, null); + } + } + + protected void addOnlyPublisherVideo(OpenViduTestappUser user, boolean simulcast, boolean dynacast, boolean hd, + String scalabilityMode) + throws InterruptedException { + if (hd) { + this.addPublisher(user, false, simulcast, dynacast, false, false, true, 1920, 1080, scalabilityMode); + } else { + this.addPublisher(user, false, simulcast, dynacast, false, false, true, null, null, null); + } + } + + protected void addOnlyPublisherAudio(OpenViduTestappUser user) throws InterruptedException { + this.addPublisher(user, false, false, false, false, true, false, null, null, null); + } + + protected void addPublisher(OpenViduTestappUser user, boolean isSubscriber, boolean simulcast, boolean dynacast, + boolean adaptiveStream, boolean hasAudio, boolean hasVideo, Integer width, Integer height, + String scalabilityMode) throws InterruptedException { + if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { + user.getDriver().findElement(By.id("close-dialog-btn")).click(); + Thread.sleep(300); + } + final int previousInstances = user.getDriver().findElements(By.cssSelector("app-openvidu-instance")).size(); + user.getDriver().findElement(By.id("add-user-btn")).click(); + // The new instance is rendered asynchronously: counting the instances right + // after the click can still see only the previous ones, and then every + // "#openvidu-instance-" selector built from that count is off by one + // (with a single instance it even becomes "#openvidu-instance--1") + user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.cssSelector("app-openvidu-instance"), + previousInstances + 1)); + int numberOfUser = previousInstances; + if (!isSubscriber) { + user.getDriver().findElement(By.cssSelector("#openvidu-instance-" + numberOfUser + " .subscriber-checkbox")) + .click(); + } + this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); + Thread.sleep(300); + if (!hasAudio) { + user.getDriver().findElement(By.id("audio-capture-false")).click(); + } else { + user.getDriver().findElement(By.id("audio-capture-true")).click(); + } + if (!hasVideo) { + user.getDriver().findElement(By.id("video-capture-false")).click(); + } else { + user.getDriver().findElement(By.id("video-capture-true")).click(); + if (width != null || height != null || scalabilityMode != null) { + this.setPublisherCustomVideoProperties(user, width, height, scalabilityMode); + } + } + if (!simulcast) { + user.getDriver().findElement(By.id("trackPublish-simulcast")).click(); + } + if (!dynacast) { + user.getDriver().findElement(By.id("room-dynacast")).click(); + } + if (!adaptiveStream) { + user.getDriver().findElement(By.id("room-adaptiveStream")).click(); + } + user.getDriver().findElement(By.id("close-dialog-btn")).click(); + Thread.sleep(300); + } + + protected void addSubscriber(OpenViduTestappUser user, boolean adaptiveStream) throws InterruptedException { + if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { + user.getDriver().findElement(By.id("close-dialog-btn")).click(); + Thread.sleep(300); + } + final int previousInstances = user.getDriver().findElements(By.cssSelector("app-openvidu-instance")).size(); + user.getDriver().findElement(By.id("add-user-btn")).click(); + // The new instance is rendered asynchronously: counting the instances right + // after the click can still see only the previous ones, and then every + // "#openvidu-instance-" selector built from that count is off by one + // (with a single instance it even becomes "#openvidu-instance--1") + user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.cssSelector("app-openvidu-instance"), + previousInstances + 1)); + int numberOfUser = previousInstances; + user.getDriver().findElement(By.cssSelector("#openvidu-instance-" + numberOfUser + " .publisher-checkbox")) + .click(); + if (!adaptiveStream) { + this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); + this.waitForBackdropAndClick(user, "#room-adaptiveStream"); + user.getDriver().findElement(By.id("close-dialog-btn")).click(); + Thread.sleep(300); + } + } + + protected void createIngress(OpenViduTestappUser user, String preset, String codec, boolean simulcast, + String urlType, + String urlUri) throws InterruptedException { + if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { + this.waitForBackdropAndClick(user, "#close-dialog-btn"); + Thread.sleep(300); + } + user.getDriver().findElement(By.xpath("//button[contains(@title,'Room API')]")).click(); + if (preset != null) { + this.waitForBackdropAndClick(user, "#ingress-preset-select"); + this.waitForBackdropAndClick(user, "#mat-option-" + preset.toUpperCase()); + } else { + if (!simulcast) { + this.waitForBackdropAndClick(user, "#ingress-simulcast"); + Thread.sleep(300); + } + this.waitForBackdropAndClick(user, "#ingress-video-codec-select"); + this.waitForBackdropAndClick(user, "#mat-option-" + codec.toUpperCase()); + } + if (urlType != null) { + this.waitForBackdropAndClick(user, "#ingress-url-type-select"); + this.waitForBackdropAndClick(user, "#mat-option-" + urlType.toUpperCase()); + } + if (urlUri != null) { + user.getDriver().findElement(By.cssSelector("#ingress-url-uri-field")).sendKeys(urlUri); + Thread.sleep(300); + } + this.waitForBackdropAndClick(user, "#create-ingress-api-btn"); + this.waitForBackdropAndClick(user, "#close-dialog-btn"); + Thread.sleep(300); + } + + protected void setPublisherSimulcastLayersAndResolution(OpenViduTestappUser user, int numberOfUser, + String simulcastLayerName, Integer width, Integer height) throws InterruptedException { + this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); + Thread.sleep(300); + this.setPublisherCustomVideoProperties(user, width, height, null); + user.getDriver().findElement(By.id("trackPublish-videoSimulcastLayers")).click(); + this.waitForBackdropAndClick(user, "#mat-option-" + simulcastLayerName); + new org.openqa.selenium.interactions.Actions(user.getDriver()) + .sendKeys(org.openqa.selenium.Keys.ESCAPE).perform(); + Thread.sleep(300); + this.waitForBackdropAndClick(user, "#close-dialog-btn"); + Thread.sleep(300); + } + + protected void setPublisherCustomVideoProperties(OpenViduTestappUser user, Integer width, Integer height, + String scalabilityMode) { + user.getDriver().findElement(By.id("video-capture-custom")).click(); + if (width != null) { + WebElement trackWidth = user.getDriver().findElement(By.id("resolution-video-capture-options-width")); + trackWidth.clear(); + trackWidth.sendKeys(width.toString()); + } + if (height != null) { + WebElement trackHeight = user.getDriver().findElement(By.id("resolution-video-capture-options-height")); + trackHeight.clear(); + trackHeight.sendKeys(height.toString()); + } + if (scalabilityMode != null) { + user.getDriver().findElement(By.id("trackPublish-scalabilityMode")).click(); + this.waitForBackdropAndClick(user, ".mode-" + scalabilityMode); + } + } + + /** + * Waits for any Material Design backdrop overlays to disappear and then clicks + * the element. This prevents ElementClickInterceptedException caused by overlay + * backdrops. + */ + protected void waitForBackdropAndClick(OpenViduTestappUser user, String cssSelector) { + final long startTime = System.currentTimeMillis(); + final long timeoutMillis = 10000; // 10 seconds total timeout + final long retryIntervalMillis = 500; // 500ms between retries + + WebElement element = null; + + while (System.currentTimeMillis() - startTime < timeoutMillis) { + try { + // Try to find and click the element immediately + element = user.getDriver().findElement(By.cssSelector(cssSelector)); + if (element.isDisplayed() && element.isEnabled()) { + element.click(); + return; // Success! Exit the method + } + } catch (org.openqa.selenium.ElementClickInterceptedException e) { + // Element is being intercepted by overlay, continue retrying + } catch (org.openqa.selenium.NoSuchElementException e) { + // Element not found, wait a bit and retry + } catch (org.openqa.selenium.StaleElementReferenceException e) { + // Element reference is stale, retry with fresh element + } catch (Exception e) { + // Any other exception, continue retrying + } + + // Wait before next retry + try { + Thread.sleep(retryIntervalMillis); + } catch (InterruptedException e) { + // Print screenshot + String screenshot = "data:image/png;base64," + + ((TakesScreenshot) user.getDriver()).getScreenshotAs(BASE64); + System.out.println("INTERRUPTED EXCEPTION WHILE WAITING FOR ELEMENT TO BE CLICKABLE: " + cssSelector); + System.out.println(screenshot); + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread interrupted while waiting for backdrop to clear", e); + } + } + + String screenshot = "data:image/png;base64," + ((TakesScreenshot) user.getDriver()).getScreenshotAs(BASE64); + System.out.println("TIMEOUT WAITING FOR ELEMENT TO BE CLICKABLE (): " + cssSelector); + System.out.println(screenshot); + + // If we get here, we've timed out + throw new RuntimeException("Timeout waiting for element '" + cssSelector + + "' to be clickable without backdrop interference after " + timeoutMillis + "ms"); + } + + protected boolean assertAllElementsHaveTracks(OpenViduTestappUser user, String selector, boolean hasAudio, + boolean hasVideo) { + org.openqa.selenium.JavascriptExecutor js = (org.openqa.selenium.JavascriptExecutor) user.getDriver(); + String script = "var elements = document.querySelectorAll(arguments[0]);" + + "for (var i = 0; i < elements.length; i++) {" + + " var el = elements[i];" + + " if (!el.srcObject) return false;" + + " if (arguments[1] && el.srcObject.getAudioTracks().length === 0) return false;" + + " if (!arguments[1] && el.srcObject.getAudioTracks().length > 0) return false;" + + " if (arguments[2] && el.srcObject.getVideoTracks().length === 0) return false;" + + " if (!arguments[2] && el.srcObject.getVideoTracks().length > 0) return false;" + + "}" + + "return true;"; + return (Boolean) js.executeScript(script, selector, hasAudio, hasVideo); + } + + protected void changeElementSize(OpenViduTestappUser user, org.openqa.selenium.WebElement element, int width, + int height) { + org.openqa.selenium.JavascriptExecutor js = (org.openqa.selenium.JavascriptExecutor) user.getDriver(); + js.executeScript( + "arguments[0].style.width = '" + width + "px'; arguments[0].style.height = '" + height + "px';", + element); + } + + /** + * Waits until the subscriber's video bytesReceived grows between two + * consecutive samples. Unlike waitUntilSubscriberBytesReceivedIncrease (which + * compares against the very first sample) this tolerates the periodic + * inbound-rtp counter restarts Firefox shows with the mediasoup engine: a + * low-bitrate stream could otherwise never climb back above a first sample + * taken late in a counter window. + */ + protected void waitUntilSubscriberBytesReceivedIncreasing(OpenViduTestappUser user, WebElement videoElement) { + final java.util.concurrent.atomic.AtomicLong previous = new java.util.concurrent.atomic.AtomicLong( + this.getSubscriberVideoBytesReceived(user, videoElement)); + this.waitUntilAux(user, videoElement, () -> { + long current = this.getSubscriberVideoBytesReceived(user, videoElement); + return current > previous.getAndSet(current); + }, "Timeout waiting for the subscriber track bytesReceived to grow between consecutive samples"); + } } diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java new file mode 100644 index 000000000..1bfd952f9 --- /dev/null +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eServerSdkTest.java @@ -0,0 +1,357 @@ +package io.openvidu.test.e2e; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.openqa.selenium.By; +import org.openqa.selenium.JavascriptExecutor; +import org.openqa.selenium.Keys; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.google.gson.JsonArray; +import com.google.gson.JsonParser; + +import livekit.LivekitModels.ParticipantInfo; +import livekit.LivekitModels.TrackInfo; +import livekit.LivekitModels.TrackType; +import livekit.LivekitModels.VideoLayer; + +/** + * E2E tests of the LiveKit server RTC SDK publishers (go, node, python, rust, + * dotnet) against browser subscribers. + * + * Split out of {@link OpenViduTestAppE2eTest} because the full matrix is slow: + * this way it can be run on its own with + * {@code mvn -Dtest=OpenViduTestAppE2eServerSdkTest test}, and the system + * properties {@code sdk.codecs} / {@code sdk.layers} narrow it further. + * + * @author Pablo Fuente (pablofuenteperez@gmail.com) + */ +@Tag("e2e") +@DisplayName("E2E tests for OpenVidu TestApp: server SDK publishers") +@ExtendWith(SpringExtension.class) +public class OpenViduTestAppE2eServerSdkTest extends AbstractOpenViduTestappE2eTest { + + @BeforeAll() + protected static void setupAll() throws Exception { + loadEnvironmentVariables(); + setUpLiveKitClient(); + CompletableFuture.runAsync(OpenViduTestAppE2eServerSdkTest::pullRemoteBrowserImages); + } + + @BeforeEach() + protected void setupEach() { + this.closeAllRooms(LK); + } + + @AfterEach() + protected void finishEach() { + this.closeAllRooms(LK); + } + + // Server RTC SDK publishers matrix: each SDK publishes each codec first as + // one plain RTP encoding (no simulcast for VP8/H264, no SVC for VP9/AV1) + // and then as two layers (high and low quality: simulcast for VP8/H264, SVC + // L2T2 for VP9/AV1), and a Chrome subscriber and a Firefox subscriber must + // both receive it — for two layers, switching to the LOW and then to the + // HIGH layer. The Go SDK single-layer H264 lane reproduces the mediasoup + // Producer codec-binding bug: Go SDK publishers declare no codec in their + // AddTrackRequest and prefer an H264 variant the server does not support, + // so the server's answer ends up VP8 first, the Producer is bound to VP8, + // the worker discards every incoming packet and the subscribers receive no + // media at all. See MEDIASOUP_CODEC_BINDING_BUG.md + + /** + * {vp8, h264, vp9, av1} x {single, multi} layers, single-layer cases first. + * System properties sdk.codecs / sdk.layers (comma-separated) restrict the + * matrix, e.g. -Dsdk.layers=multi -Dsdk.codecs=vp9,av1 + */ + static Stream serverSdkPublisherMatrix() { + List layersFilter = List.of(System.getProperty("sdk.layers", "single,multi").split(",")); + List codecsFilter = List.of(System.getProperty("sdk.codecs", "vp8,h264,vp9,av1").split(",")); + return Stream.of("single", "multi").filter(layersFilter::contains) + .flatMap(layers -> Stream.of("vp8", "h264", "vp9", "av1").filter(codecsFilter::contains) + .map(codec -> Arguments.of(codec, layers))); + } + + @ParameterizedTest(name = "Go SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") + @MethodSource("serverSdkPublisherMatrix") + @DisplayName("Go SDK publisher to Chrome and Firefox subscribers") + void goSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { + serverSdkPublisherToBrowserSubscribersAux("go", codec, layers); + } + + @ParameterizedTest(name = "Node SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") + @MethodSource("serverSdkPublisherMatrix") + @DisplayName("Node SDK publisher to Chrome and Firefox subscribers") + void nodeSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { + serverSdkPublisherToBrowserSubscribersAux("node", codec, layers); + } + + @ParameterizedTest(name = "Python SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") + @MethodSource("serverSdkPublisherMatrix") + @DisplayName("Python SDK publisher to Chrome and Firefox subscribers") + void pythonSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { + serverSdkPublisherToBrowserSubscribersAux("python", codec, layers); + } + + @ParameterizedTest(name = "Rust SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") + @MethodSource("serverSdkPublisherMatrix") + @DisplayName("Rust SDK publisher to Chrome and Firefox subscribers") + void rustSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { + serverSdkPublisherToBrowserSubscribersAux("rust", codec, layers); + } + + @ParameterizedTest(name = ".NET SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") + @MethodSource("serverSdkPublisherMatrix") + @DisplayName(".NET SDK publisher to Chrome and Firefox subscribers") + void dotnetSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { + // Requires Livekit.Rtc.Dotnet >= 0.1.4 (TrackPublishOptions.VideoCodec) + serverSdkPublisherToBrowserSubscribersAux("dotnet", codec, layers); + } + + /** + * A Chrome browser and a Firefox browser join the room as subscriber-only + * participants and a LiveKit server RTC SDK participant + * (startServerSdkPublisher) joins the same room ("TestRoom" is the testapp + * default) publishing a single video track with the given codec: as one plain + * RTP encoding (layers "single") or as two layers (layers "multi": high and + * low quality — simulcast for VP8/H264, SVC L2T2 for VP9/AV1). Both + * subscribers must receive the track's media with that codec, going through + * exactly the same steps; with two layers they also switch to the LOW and + * then to the HIGH layer. + */ + private void serverSdkPublisherToBrowserSubscribersAux(String sdk, String codec, String layers) + throws Exception { + final String expectedCodec = "video/" + codec.toUpperCase(); + final String publisherIdentity = sdk + "-publisher"; + final boolean multiLayer = "multi".equals(layers); + + // The Go SDK forwards pre-encoded samples (no encoder), so its only + // multi-layer shape is RID simulcast — and LiveKit does not support RID + // simulcast for the SVC-class codecs (multi-layer VP9/AV1 must be SVC): + // that combination is skipped as an unsupported publish shape + Assumptions.assumeFalse(multiLayer && "go".equals(sdk) && ("vp9".equals(codec) || "av1".equals(codec)), + "The Go SDK cannot publish SVC, and VP9/AV1 RID simulcast is not a supported LiveKit publish shape"); + + List subscribers = List.of(setupBrowserAndConnectToOpenViduTestapp("chrome"), + setupBrowserAndConnectToOpenViduTestapp("firefox")); + + log.info("{} SDK {} {}-layer publisher to Chrome and Firefox subscribers", sdk, codec, layers); + + // Both browsers join the room as subscriber-only participants with + // adaptiveStream disabled (the received layer only changes through the + // testapp's max-video-quality selector), each with its own identity (a + // second join with the testapp's default identity would kick the first + // browser out of the room) + for (OpenViduTestappUser user : subscribers) { + this.addSubscriber(user, false); + WebElement participantNameInput = user.getDriver().findElement(By.id("participant-name-input-0")); + participantNameInput.clear(); + participantNameInput.sendKeys(browserName(user) + "-subscriber"); + user.getDriver().findElements(By.className("connect-btn")).forEach(el -> el.sendKeys(Keys.ENTER)); + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 1); + } + + this.startServerSdkPublisher(sdk, "TestRoom", codec, multiLayer); + + // The server's authoritative view of the publication (RoomService API): + // the LiveKit TrackInfo of a video publication lists one layer per + // simulcast layer or SVC spatial layer, so one plain encoding (no + // simulcast, no SVC / L1T1) has exactly one layer and a two-layer + // publish (simulcast or SVC L2T2) has two + TrackInfo trackInfo = this.getPublishedVideoTrackInfo("TestRoom", publisherIdentity); + final int expectedLayers = multiLayer ? 2 : 1; + Assertions.assertEquals(expectedLayers, trackInfo.getLayersCount(), "Expected " + expectedLayers + + " video layer(s) in the track published by the " + sdk + " SDK, but the server reports " + + trackInfo.getLayersList()); + Assertions.assertEquals(expectedLayers, trackInfo.getCodecs(0).getLayersCount(), "Expected " + expectedLayers + + " video layer(s) for codec " + expectedCodec + " published by the " + sdk + " SDK"); + // VP8/H264 multi-layer publishes are simulcast and VP9/AV1 ones are SVC + // L2T2, except for the Go SDK: it forwards pre-encoded samples (no + // encoder, so no SVC) and publishes simulcast for every codec + final boolean expectSimulcast = multiLayer && ("vp8".equals(codec) || "h264".equals(codec) || "go".equals(sdk)); + if (!multiLayer || expectSimulcast) { + Assertions.assertEquals(expectSimulcast, trackInfo.getSimulcast(), + "Simulcast flag of the track published by the " + sdk + " SDK"); + // (the server's explicit SVC verdict: an SVC publication would be + // MULTIPLE_SPATIAL_LAYERS_PER_STREAM) + Assertions.assertEquals(VideoLayer.Mode.ONE_SPATIAL_LAYER_PER_STREAM, + trackInfo.getCodecs(0).getVideoLayerMode(), + "The track published by the " + sdk + " SDK should not be SVC"); + } + + // Both browsers must receive the track, with its codec and its layers + for (OpenViduTestappUser user : subscribers) { + assertSubscriberReceivesVideo(user, sdk, publisherIdentity, expectedCodec, trackInfo); + } + + for (OpenViduTestappUser user : subscribers) { + gracefullyLeaveParticipants(user, 1); + } + } + + /** "Chrome", "Firefox"... from the BrowserUser class of the testapp user. */ + private String browserName(OpenViduTestappUser user) { + return user.getBrowserUser().getClass().getSimpleName().replace("User", ""); + } + + /** + * Selects the max video quality (LOW, MEDIUM or HIGH) of the remote track of + * the first testapp instance, closing the track info dialog if it is open. + */ + private void selectSubscriberVideoQuality(OpenViduTestappUser user, String quality) throws InterruptedException { + if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) { + user.getDriver().findElement(By.cssSelector("#close-dialog-btn")).click(); + Thread.sleep(300); + } + user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click(); + this.waitForBackdropAndClick(user, "mat-option.mode-" + quality); + } + + /** + * The subscriber-only participant of the given browser must receive the video + * track published by the SDK participant: media actually flowing, with the + * expected codec, with the layers the server reports in trackInfo. With more + * than one layer the subscriber switches to the LOW and then to the HIGH + * layer, each recognised by its frame width. + */ + private void assertSubscriberReceivesVideo(OpenViduTestappUser user, String sdk, String publisherIdentity, + String expectedCodec, TrackInfo trackInfo) throws Exception { + final String browser = browserName(user); + + user.getEventManager().waitUntilEventReaches("trackSubscribed", "ParticipantEvent", 1); + + user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.tagName("video"), 1)); + Assertions.assertTrue(assertAllElementsHaveTracks(user, "video", false, true), + browser + ": HTMLVideoElements were expected to have only one video track"); + + WebElement subscriberVideo = user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 video.remote")); + + // Media must actually reach the subscriber (with the codec-binding bug + // present the track subscribes but receives 0 bytes forever) + waitUntilVideoLayersNotEmpty(user, subscriberVideo); + this.waitUntilSubscriberBytesReceivedIncreasing(user, subscriberVideo); + this.waitUntilSubscriberFramesPerSecondNotZero(user, subscriberVideo); + + // And with the codec the publisher actually sends (a Producer bound to + // the wrong codec makes every subscriber negotiate that wrong codec) + Assertions.assertEquals(expectedCodec, this.getSubscriberVideoCodec(user, subscriberVideo), + browser + " subscriber should negotiate the codec the " + sdk + " SDK publisher sends"); + + // And with the layers the server reports, in the track info received by + // the subscriber + JsonArray subscriberLayers = this.getRemoteVideoTrackInfoLayers(user, publisherIdentity); + Assertions.assertEquals(trackInfo.getLayersCount(), subscriberLayers.size(), + "Expected " + trackInfo.getLayersCount() + " video layer(s) in the track published by the " + sdk + + " SDK, but the " + browser + " subscriber sees " + subscriberLayers); + + if (trackInfo.getLayersCount() > 1) { + // Multiple layers: with adaptiveStream disabled the received layer + // only changes through the max-video-quality selector. Receiving the + // LOW layer and then the HIGH layer — each recognised by the frame + // width the publisher declared for it in the TrackInfo, and each + // actually decoding (framesPerSecond > 0: an undecodable layer still + // reports its frame size) — proves that the publisher sends every + // layer and that the SFU forwards the requested one. The declared + // widths are reliable because the multi-layer publishers capture at + // 15 fps: at 30 fps libwebrtc's CPU adaptation can scale every layer + // down under load + this.selectQualityAndAwaitLayer(user, subscriberVideo, "LOW", lowestLayerWidth(trackInfo)); + this.selectQualityAndAwaitLayer(user, subscriberVideo, "HIGH", highestLayerWidth(trackInfo)); + } + } + + /** + * Selects the max video quality of the subscriber's remote track and waits + * until the received video has the frame width of that layer and decodes + * (framesPerSecond > 0). Retries the selection once: under CPU load the SFU + * can take longer than one wait window to ramp back up to a higher layer. + */ + private void selectQualityAndAwaitLayer(OpenViduTestappUser user, WebElement subscriberVideo, String quality, + int expectedFrameWidth) throws Exception { + for (int attempt = 1; attempt <= 2; attempt++) { + this.selectSubscriberVideoQuality(user, quality); + try { + this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, expectedFrameWidth); + break; + } catch (AssertionError e) { + if (attempt == 2) { + throw e; + } + } + } + this.waitUntilSubscriberFramesPerSecondNotZero(user, subscriberVideo); + } + + /** + * Width of the lowest-quality published layer, from the server's TrackInfo. + * The layers are picked by width, not by their VideoQuality label: the SDKs + * label a two-layer publish inconsistently (simulcast: LOW + MEDIUM; SVC + * L2T2: MEDIUM + HIGH; the Go program: LOW + HIGH), while the subscriber's + * LOW/HIGH selection always clamps to the lowest/highest available layer. + */ + private int lowestLayerWidth(TrackInfo trackInfo) { + return trackInfo.getLayersList().stream().mapToInt(VideoLayer::getWidth).min() + .orElseThrow(() -> new AssertionError("No layers in " + trackInfo)); + } + + /** + * Width of the highest-quality published layer, from the server's TrackInfo. + */ + private int highestLayerWidth(TrackInfo trackInfo) { + return trackInfo.getLayersList().stream().mapToInt(VideoLayer::getWidth).max() + .orElseThrow(() -> new AssertionError("No layers in " + trackInfo)); + } + + /** + * The first video track published by the given participant, as reported by + * the LiveKit server (RoomService GetParticipant). Waits up to 10 seconds for + * it: the SDK programs log TRACK_PUBLISHED as soon as their publish call + * returns, a few milliseconds before the server registers the track. + */ + private TrackInfo getPublishedVideoTrackInfo(String roomName, String participantIdentity) throws Exception { + for (int attempt = 0; attempt < 40; attempt++) { + ParticipantInfo participant = LK.getParticipant(roomName, participantIdentity).execute().body(); + if (participant != null) { + Optional videoTrack = participant.getTracksList().stream() + .filter(track -> track.getType() == TrackType.VIDEO).findFirst(); + if (videoTrack.isPresent()) { + return videoTrack.get(); + } + } + Thread.sleep(250); + } + throw new AssertionError(participantIdentity + " has no published video track in room " + roomName); + } + + /** + * Video layers (VideoLayer[] of the LiveKit TrackInfo) of the first video + * track published by the remote participant with the given identity, as seen + * by the local participant of the first testapp instance. + */ + private JsonArray getRemoteVideoTrackInfoLayers(OpenViduTestappUser user, String participantIdentity) { + String layers = (String) ((JavascriptExecutor) user.getDriver()).executeScript( + "var room = window['room_0'];" + + "var participant = room.remoteParticipants.get(arguments[0]);" + + "var publication = participant.videoTrackPublications.values().next().value;" + + "return JSON.stringify(publication.trackInfo.layers);", + participantIdentity); + return JsonParser.parseString(layers).getAsJsonArray(); + } +} diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java index d775b7397..4ee33ac29 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java @@ -27,8 +27,6 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; @@ -45,7 +43,6 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -53,14 +50,9 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import org.openqa.selenium.By; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.Keys; -import org.openqa.selenium.TakesScreenshot; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.support.ui.WebDriverWait; @@ -86,12 +78,6 @@ import io.minio.messages.Item; import livekit.LivekitIngress.IngressInfo; import livekit.LivekitIngress.IngressState; import livekit.LivekitModels.ConnectionQuality; -import livekit.LivekitModels.ParticipantInfo; -import livekit.LivekitModels.TrackInfo; -import livekit.LivekitModels.TrackType; -import livekit.LivekitModels.VideoLayer; - -import static org.openqa.selenium.OutputType.BASE64; /** * E2E tests for openvidu-testapp. @@ -104,14 +90,6 @@ import static org.openqa.selenium.OutputType.BASE64; @ExtendWith(SpringExtension.class) public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { - private static final long WAIT_UNTIL_MAX_MILLIS = 20000; - - // Minimum average frame rate that a subscriber video must sustain over a - // window of at least MIN_FRAMES_DECODED_WINDOW_MILLIS to be considered - // properly decoded and played - private static final long MIN_FRAMES_DECODED_FPS = 4; - private static final long MIN_FRAMES_DECODED_WINDOW_MILLIS = 2000; - @BeforeAll() protected static void setupAll() throws Exception { checkFfmpegInstallation(); @@ -127,24 +105,6 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { CompletableFuture.runAsync(OpenViduTestAppE2eTest::pullRemoteBrowserImages); } - private static void pullRemoteBrowserImages() { - pullRemoteBrowserImage("REMOTE_URL_CHROME", "selenium/standalone-chrome:" + CHROME_VERSION); - pullRemoteBrowserImage("REMOTE_URL_FIREFOX", "selenium/standalone-firefox:" + FIREFOX_VERSION); - pullRemoteBrowserImage("REMOTE_URL_EDGE", "selenium/standalone-edge:" + EDGE_VERSION); - } - - private static void pullRemoteBrowserImage(String remoteUrlProperty, String image) { - if (System.getProperty(remoteUrlProperty) == null) { - return; // This browser runs as a native driver here. No Docker image to pull - } - try { - log.info("Pre-pulling Selenium image {}", image); - commandLine.executeCommand("docker pull " + image, 300); - } catch (Exception e) { - System.err.println("Pre-pull of " + image + " failed: " + e.getMessage()); - } - } - @BeforeEach() protected void setupEach() { this.closeAllRooms(LK); @@ -4031,314 +3991,6 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { testNoSimulcast(user, subscriberVideo); } - // Server RTC SDK publishers matrix: each SDK publishes each codec first as - // one plain RTP encoding (no simulcast for VP8/H264, no SVC for VP9/AV1) - // and then as two layers (high and low quality: simulcast for VP8/H264, SVC - // L2T2 for VP9/AV1), and a Chrome subscriber and a Firefox subscriber must - // both receive it — for two layers, switching to the LOW and then to the - // HIGH layer. The Go SDK single-layer H264 lane reproduces the mediasoup - // Producer codec-binding bug: Go SDK publishers declare no codec in their - // AddTrackRequest and prefer an H264 variant the server does not support, - // so the server's answer ends up VP8 first, the Producer is bound to VP8, - // the worker discards every incoming packet and the subscribers receive no - // media at all. See MEDIASOUP_CODEC_BINDING_BUG.md - - /** - * {vp8, h264, vp9, av1} x {single, multi} layers, single-layer cases first. - * System properties sdk.codecs / sdk.layers (comma-separated) restrict the - * matrix, e.g. -Dsdk.layers=multi -Dsdk.codecs=vp9,av1 - */ - static Stream serverSdkPublisherMatrix() { - List layersFilter = List.of(System.getProperty("sdk.layers", "single,multi").split(",")); - List codecsFilter = List.of(System.getProperty("sdk.codecs", "vp8,h264,vp9,av1").split(",")); - return Stream.of("single", "multi").filter(layersFilter::contains) - .flatMap(layers -> Stream.of("vp8", "h264", "vp9", "av1").filter(codecsFilter::contains) - .map(codec -> Arguments.of(codec, layers))); - } - - @ParameterizedTest(name = "Go SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") - @MethodSource("serverSdkPublisherMatrix") - @DisplayName("Go SDK publisher to Chrome and Firefox subscribers") - void goSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { - serverSdkPublisherToBrowserSubscribersAux("go", codec, layers); - } - - @ParameterizedTest(name = "Node SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") - @MethodSource("serverSdkPublisherMatrix") - @DisplayName("Node SDK publisher to Chrome and Firefox subscribers") - void nodeSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { - serverSdkPublisherToBrowserSubscribersAux("node", codec, layers); - } - - @ParameterizedTest(name = "Python SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") - @MethodSource("serverSdkPublisherMatrix") - @DisplayName("Python SDK publisher to Chrome and Firefox subscribers") - void pythonSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { - serverSdkPublisherToBrowserSubscribersAux("python", codec, layers); - } - - @ParameterizedTest(name = "Rust SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") - @MethodSource("serverSdkPublisherMatrix") - @DisplayName("Rust SDK publisher to Chrome and Firefox subscribers") - void rustSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { - serverSdkPublisherToBrowserSubscribersAux("rust", codec, layers); - } - - @ParameterizedTest(name = ".NET SDK {0} {1}-layer publisher to Chrome and Firefox subscribers") - @MethodSource("serverSdkPublisherMatrix") - @DisplayName(".NET SDK publisher to Chrome and Firefox subscribers") - void dotnetSdkPublisherToBrowserSubscribersTest(String codec, String layers) throws Exception { - // Requires Livekit.Rtc.Dotnet >= 0.1.4 (TrackPublishOptions.VideoCodec) - serverSdkPublisherToBrowserSubscribersAux("dotnet", codec, layers); - } - - /** - * A Chrome browser and a Firefox browser join the room as subscriber-only - * participants and a LiveKit server RTC SDK participant - * (startServerSdkPublisher) joins the same room ("TestRoom" is the testapp - * default) publishing a single video track with the given codec: as one plain - * RTP encoding (layers "single") or as two layers (layers "multi": high and - * low quality — simulcast for VP8/H264, SVC L2T2 for VP9/AV1). Both - * subscribers must receive the track's media with that codec, going through - * exactly the same steps; with two layers they also switch to the LOW and - * then to the HIGH layer. - */ - private void serverSdkPublisherToBrowserSubscribersAux(String sdk, String codec, String layers) - throws Exception { - final String expectedCodec = "video/" + codec.toUpperCase(); - final String publisherIdentity = sdk + "-publisher"; - final boolean multiLayer = "multi".equals(layers); - - // The Go SDK forwards pre-encoded samples (no encoder), so its only - // multi-layer shape is RID simulcast — and LiveKit does not support RID - // simulcast for the SVC-class codecs (multi-layer VP9/AV1 must be SVC): - // that combination is skipped as an unsupported publish shape - Assumptions.assumeFalse(multiLayer && "go".equals(sdk) && ("vp9".equals(codec) || "av1".equals(codec)), - "The Go SDK cannot publish SVC, and VP9/AV1 RID simulcast is not a supported LiveKit publish shape"); - - List subscribers = List.of(setupBrowserAndConnectToOpenViduTestapp("chrome"), - setupBrowserAndConnectToOpenViduTestapp("firefox")); - - log.info("{} SDK {} {}-layer publisher to Chrome and Firefox subscribers", sdk, codec, layers); - - // Both browsers join the room as subscriber-only participants with - // adaptiveStream disabled (the received layer only changes through the - // testapp's max-video-quality selector), each with its own identity (a - // second join with the testapp's default identity would kick the first - // browser out of the room) - for (OpenViduTestappUser user : subscribers) { - this.addSubscriber(user, false); - WebElement participantNameInput = user.getDriver().findElement(By.id("participant-name-input-0")); - participantNameInput.clear(); - participantNameInput.sendKeys(browserName(user) + "-subscriber"); - user.getDriver().findElements(By.className("connect-btn")).forEach(el -> el.sendKeys(Keys.ENTER)); - user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 1); - } - - this.startServerSdkPublisher(sdk, "TestRoom", codec, multiLayer); - - // The server's authoritative view of the publication (RoomService API): - // the LiveKit TrackInfo of a video publication lists one layer per - // simulcast layer or SVC spatial layer, so one plain encoding (no - // simulcast, no SVC / L1T1) has exactly one layer and a two-layer - // publish (simulcast or SVC L2T2) has two - TrackInfo trackInfo = this.getPublishedVideoTrackInfo("TestRoom", publisherIdentity); - final int expectedLayers = multiLayer ? 2 : 1; - Assertions.assertEquals(expectedLayers, trackInfo.getLayersCount(), "Expected " + expectedLayers - + " video layer(s) in the track published by the " + sdk + " SDK, but the server reports " - + trackInfo.getLayersList()); - Assertions.assertEquals(expectedLayers, trackInfo.getCodecs(0).getLayersCount(), "Expected " + expectedLayers - + " video layer(s) for codec " + expectedCodec + " published by the " + sdk + " SDK"); - // VP8/H264 multi-layer publishes are simulcast and VP9/AV1 ones are SVC - // L2T2, except for the Go SDK: it forwards pre-encoded samples (no - // encoder, so no SVC) and publishes simulcast for every codec - final boolean expectSimulcast = multiLayer && ("vp8".equals(codec) || "h264".equals(codec) || "go".equals(sdk)); - if (!multiLayer || expectSimulcast) { - Assertions.assertEquals(expectSimulcast, trackInfo.getSimulcast(), - "Simulcast flag of the track published by the " + sdk + " SDK"); - // (the server's explicit SVC verdict: an SVC publication would be - // MULTIPLE_SPATIAL_LAYERS_PER_STREAM) - Assertions.assertEquals(VideoLayer.Mode.ONE_SPATIAL_LAYER_PER_STREAM, - trackInfo.getCodecs(0).getVideoLayerMode(), - "The track published by the " + sdk + " SDK should not be SVC"); - } - - // Both browsers must receive the track, with its codec and its layers - for (OpenViduTestappUser user : subscribers) { - assertSubscriberReceivesVideo(user, sdk, publisherIdentity, expectedCodec, trackInfo); - } - - for (OpenViduTestappUser user : subscribers) { - gracefullyLeaveParticipants(user, 1); - } - } - - /** "Chrome", "Firefox"... from the BrowserUser class of the testapp user. */ - private String browserName(OpenViduTestappUser user) { - return user.getBrowserUser().getClass().getSimpleName().replace("User", ""); - } - - /** - * Waits until the subscriber's video bytesReceived grows between two - * consecutive samples. Unlike waitUntilSubscriberBytesReceivedIncrease (which - * compares against the very first sample) this tolerates the periodic - * inbound-rtp counter restarts Firefox shows with the mediasoup engine: a - * low-bitrate stream could otherwise never climb back above a first sample - * taken late in a counter window. - */ - private void waitUntilSubscriberBytesReceivedIncreasing(OpenViduTestappUser user, WebElement videoElement) { - final java.util.concurrent.atomic.AtomicLong previous = new java.util.concurrent.atomic.AtomicLong( - this.getSubscriberVideoBytesReceived(user, videoElement)); - this.waitUntilAux(user, videoElement, () -> { - long current = this.getSubscriberVideoBytesReceived(user, videoElement); - return current > previous.getAndSet(current); - }, "Timeout waiting for the subscriber track bytesReceived to grow between consecutive samples"); - } - - /** - * Selects the max video quality (LOW, MEDIUM or HIGH) of the remote track of - * the first testapp instance, closing the track info dialog if it is open. - */ - private void selectSubscriberVideoQuality(OpenViduTestappUser user, String quality) throws InterruptedException { - if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) { - user.getDriver().findElement(By.cssSelector("#close-dialog-btn")).click(); - Thread.sleep(300); - } - user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 #max-video-quality")).click(); - this.waitForBackdropAndClick(user, "mat-option.mode-" + quality); - } - - /** - * The subscriber-only participant of the given browser must receive the video - * track published by the SDK participant: media actually flowing, with the - * expected codec, with the layers the server reports in trackInfo. With more - * than one layer the subscriber switches to the LOW and then to the HIGH - * layer, each recognised by its frame width. - */ - private void assertSubscriberReceivesVideo(OpenViduTestappUser user, String sdk, String publisherIdentity, - String expectedCodec, TrackInfo trackInfo) throws Exception { - final String browser = browserName(user); - - user.getEventManager().waitUntilEventReaches("trackSubscribed", "ParticipantEvent", 1); - - user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.tagName("video"), 1)); - Assertions.assertTrue(assertAllElementsHaveTracks(user, "video", false, true), - browser + ": HTMLVideoElements were expected to have only one video track"); - - WebElement subscriberVideo = user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 video.remote")); - - // Media must actually reach the subscriber (with the codec-binding bug - // present the track subscribes but receives 0 bytes forever) - waitUntilVideoLayersNotEmpty(user, subscriberVideo); - this.waitUntilSubscriberBytesReceivedIncreasing(user, subscriberVideo); - this.waitUntilSubscriberFramesPerSecondNotZero(user, subscriberVideo); - - // And with the codec the publisher actually sends (a Producer bound to - // the wrong codec makes every subscriber negotiate that wrong codec) - Assertions.assertEquals(expectedCodec, this.getSubscriberVideoCodec(user, subscriberVideo), - browser + " subscriber should negotiate the codec the " + sdk + " SDK publisher sends"); - - // And with the layers the server reports, in the track info received by - // the subscriber - JsonArray subscriberLayers = this.getRemoteVideoTrackInfoLayers(user, publisherIdentity); - Assertions.assertEquals(trackInfo.getLayersCount(), subscriberLayers.size(), - "Expected " + trackInfo.getLayersCount() + " video layer(s) in the track published by the " + sdk - + " SDK, but the " + browser + " subscriber sees " + subscriberLayers); - - if (trackInfo.getLayersCount() > 1) { - // Multiple layers: with adaptiveStream disabled the received layer - // only changes through the max-video-quality selector. Receiving the - // LOW layer and then the HIGH layer — each recognised by the frame - // width the publisher declared for it in the TrackInfo, and each - // actually decoding (framesPerSecond > 0: an undecodable layer still - // reports its frame size) — proves that the publisher sends every - // layer and that the SFU forwards the requested one. The declared - // widths are reliable because the multi-layer publishers capture at - // 15 fps: at 30 fps libwebrtc's CPU adaptation can scale every layer - // down under load - this.selectQualityAndAwaitLayer(user, subscriberVideo, "LOW", lowestLayerWidth(trackInfo)); - this.selectQualityAndAwaitLayer(user, subscriberVideo, "HIGH", highestLayerWidth(trackInfo)); - } - } - - /** - * Selects the max video quality of the subscriber's remote track and waits - * until the received video has the frame width of that layer and decodes - * (framesPerSecond > 0). Retries the selection once: under CPU load the SFU - * can take longer than one wait window to ramp back up to a higher layer. - */ - private void selectQualityAndAwaitLayer(OpenViduTestappUser user, WebElement subscriberVideo, String quality, - int expectedFrameWidth) throws Exception { - for (int attempt = 1; attempt <= 2; attempt++) { - this.selectSubscriberVideoQuality(user, quality); - try { - this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, expectedFrameWidth); - break; - } catch (AssertionError e) { - if (attempt == 2) { - throw e; - } - } - } - this.waitUntilSubscriberFramesPerSecondNotZero(user, subscriberVideo); - } - - /** - * Width of the lowest-quality published layer, from the server's TrackInfo. - * The layers are picked by width, not by their VideoQuality label: the SDKs - * label a two-layer publish inconsistently (simulcast: LOW + MEDIUM; SVC - * L2T2: MEDIUM + HIGH; the Go program: LOW + HIGH), while the subscriber's - * LOW/HIGH selection always clamps to the lowest/highest available layer. - */ - private int lowestLayerWidth(TrackInfo trackInfo) { - return trackInfo.getLayersList().stream().mapToInt(VideoLayer::getWidth).min() - .orElseThrow(() -> new AssertionError("No layers in " + trackInfo)); - } - - /** - * Width of the highest-quality published layer, from the server's TrackInfo. - */ - private int highestLayerWidth(TrackInfo trackInfo) { - return trackInfo.getLayersList().stream().mapToInt(VideoLayer::getWidth).max() - .orElseThrow(() -> new AssertionError("No layers in " + trackInfo)); - } - - /** - * The first video track published by the given participant, as reported by - * the LiveKit server (RoomService GetParticipant). Waits up to 10 seconds for - * it: the SDK programs log TRACK_PUBLISHED as soon as their publish call - * returns, a few milliseconds before the server registers the track. - */ - private TrackInfo getPublishedVideoTrackInfo(String roomName, String participantIdentity) throws Exception { - for (int attempt = 0; attempt < 40; attempt++) { - ParticipantInfo participant = LK.getParticipant(roomName, participantIdentity).execute().body(); - if (participant != null) { - Optional videoTrack = participant.getTracksList().stream() - .filter(track -> track.getType() == TrackType.VIDEO).findFirst(); - if (videoTrack.isPresent()) { - return videoTrack.get(); - } - } - Thread.sleep(250); - } - throw new AssertionError(participantIdentity + " has no published video track in room " + roomName); - } - - /** - * Video layers (VideoLayer[] of the LiveKit TrackInfo) of the first video - * track published by the remote participant with the given identity, as seen - * by the local participant of the first testapp instance. - */ - private JsonArray getRemoteVideoTrackInfoLayers(OpenViduTestappUser user, String participantIdentity) { - String layers = (String) ((JavascriptExecutor) user.getDriver()).executeScript( - "var room = window['room_0'];" - + "var participant = room.remoteParticipants.get(arguments[0]);" - + "var publication = participant.videoTrackPublications.values().next().value;" - + "return JSON.stringify(publication.trackInfo.layers);", - participantIdentity); - return JsonParser.parseString(layers).getAsJsonArray(); - } - @Test @DisplayName("RTSP ingress H264 + OPUS") void rtspIngressH264_OPUSTest() throws Exception { @@ -4838,565 +4490,4 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { this.waitUntilSubscriberFrameWidthIs(user, subscriberVideo, 1920); } - private int countNumberOfPublishedLayers(OpenViduTestappUser user, WebElement publisherVideo) { - JsonArray json = this.getLayersAsJsonArray(user, publisherVideo); - return json.size(); - } - - private int getSubscriberVideoFrameWidth(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "frameWidth", JsonElement::getAsInt); - } - - private int getSubscriberVideoFrameHeight(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "frameHeight", JsonElement::getAsInt); - } - - private long getSubscriberVideoBytesReceived(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "bytesReceived", JsonElement::getAsLong); - } - - private int getSubscriberVideoFramesPerSecond(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "framesPerSecond", JsonElement::getAsInt); - } - - private long getSubscriberVideoFramesDecoded(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "framesDecoded", JsonElement::getAsLong); - } - - private long getSubscriberVideoFramesReceived(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "framesReceived", JsonElement::getAsLong); - } - - private String getSubscriberVideoCodec(OpenViduTestappUser user, WebElement subscriberVideo) { - return getSubscriberVideoLayerStat(user, subscriberVideo, "codec", JsonElement::getAsString); - } - - private T getSubscriberVideoLayerStat(OpenViduTestappUser user, WebElement subscriberVideo, String field, - java.util.function.Function extractor) { - final long deadline = System.currentTimeMillis() + WAIT_UNTIL_MAX_MILLIS; - JsonElement element = null; - do { - try { - element = getLayersAsJsonArray(user, subscriberVideo).get(0).getAsJsonObject().get(field); - } catch (Exception e) { - element = null; - } - if (element == null || element.isJsonNull()) { - try { - Thread.sleep(250); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } while ((element == null || element.isJsonNull()) && System.currentTimeMillis() < deadline); - if (element == null || element.isJsonNull()) { - Assertions.fail("Timeout waiting for " + field + " to exist"); - } - return extractor.apply(element); - } - - // Several stats of the same subscriber layer, taken from a single info dialog - // update. Sampling them one by one through getSubscriberVideoLayerStat would pay - // a full dialog read, and a stats refresh, for each one of them - private JsonObject getSubscriberVideoLayer(OpenViduTestappUser user, WebElement subscriberVideo) { - JsonArray layers = this.getLayersAsJsonArray(user, subscriberVideo); - return layers.isEmpty() ? new JsonObject() : layers.get(0).getAsJsonObject(); - } - - // Cumulative counter of the given layer, or -1 if it is not there - private long getLayerCounter(JsonObject layer, String field) { - JsonElement element = layer.get(field); - return element == null || element.isJsonNull() ? -1 : element.getAsLong(); - } - - // If rid is null, retrieve the first layer - private JsonElement getPublisherVideoLayerAttribute(OpenViduTestappUser user, WebElement publisherVideo, String rid, - String attribute) { - JsonArray json = this.getLayersAsJsonArray(user, publisherVideo); - JsonElement result; - if (rid != null) { - result = json.asList().stream().parallel() - .filter(jsonElement -> rid.equals(jsonElement.getAsJsonObject().get("rid").getAsString())).findAny() - .get(); - } else { - result = json.get(0); - } - return result.getAsJsonObject().get(attribute); - } - - private String getLayersAsString(OpenViduTestappUser user, WebElement video) { - this.openInfoDialog(user, video); - user.getDriver().findElement(By.cssSelector("#update-value-btn")).click(); - WebElement textarea = user.getDriver().findElement(By.id("info-text-area")); - return textarea.getAttribute("value"); - } - - private JsonArray getLayersAsJsonArray(OpenViduTestappUser user, WebElement video) { - String value = getLayersAsString(user, video); - return JsonParser.parseString(value).getAsJsonArray(); - } - - private void waitUntilVideoLayersNotEmpty(OpenViduTestappUser user, WebElement videoElement) { - this.waitUntilAux(user, videoElement, () -> { - String value = getLayersAsString(user, videoElement); - return !value.isBlank() && !JsonParser.parseString(value).getAsJsonArray().isEmpty(); - }, "Timeout waiting video layers to not be empty"); - } - - private void waitUntilSubscriberFramesPerSecondNotZero(OpenViduTestappUser user, WebElement videoElement) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoFramesPerSecond(user, videoElement) > 0; - }, "Timeout waiting for video track to have a framesPerSecond greater than 0"); - } - - private void waitUntilSubscriberFramesPerSecondIs(OpenViduTestappUser user, WebElement videoElement, int fps) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoFramesPerSecond(user, videoElement) == fps; - }, "Timeout waiting for video track to have a framesPerSecond equal to " + fps); - } - - private void waitUntilSubscriberFrameWidthIs(OpenViduTestappUser user, WebElement videoElement, - final int expectedFrameWidth) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoFrameWidth(user, videoElement) == expectedFrameWidth; - }, "Timeout waiting for video track to have a frameWidth of " + expectedFrameWidth); - } - - private void waitUntilSubscriberFrameHeightIs(OpenViduTestappUser user, WebElement videoElement, - final int expectedFrameHeight) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoFrameHeight(user, videoElement) == expectedFrameHeight; - }, "Timeout waiting for video track to have a frameHeight of " + expectedFrameHeight); - } - - private void waitUntilSubscriberFrameWidthChanges(OpenViduTestappUser user, WebElement videoElement, - final int oldFrameWidth, final boolean shouldBeHigher) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoFrameWidth(user, videoElement) != oldFrameWidth; - }, "Timeout waiting for video track to reach a " + (shouldBeHigher ? "higher" : "lower") + " resolution"); - int newFrameWidth = this.getSubscriberVideoFrameWidth(user, videoElement); - if (shouldBeHigher) { - Assertions.assertTrue(newFrameWidth > oldFrameWidth, - "Video track should have now a higher resolution, but it is not. Old width: " + oldFrameWidth - + ". New width: " + newFrameWidth); - } else { - Assertions.assertTrue(newFrameWidth < oldFrameWidth, - "Video track should have now a lower resolution, but it is not. Old width: " + oldFrameWidth - + ". New width: " + newFrameWidth); - } - } - - private void waitUntilSubscriberBytesReceivedIncrease(OpenViduTestappUser user, WebElement videoElement, - final long previousBytesReceived) { - this.waitUntilAux(user, videoElement, () -> { - return this.getSubscriberVideoBytesReceived(user, videoElement) > previousBytesReceived; - }, "Timeout waiting for subscriber track to increase its bytesReceived from " + previousBytesReceived); - } - - // A subscriber video is only properly received AND played if its decoder keeps - // producing new frames at a sustained rate. Receiving bytes is not enough: a - // subscriber may receive media that it is not able to decode at all. And a - // single new decoded frame is not enough either: a video that only decodes one - // or two frames over a timespan of several seconds is a frozen video, not a - // playing one, and must fail the test. So framesDecoded is required to grow at - // MIN_FRAMES_DECODED_FPS or more, averaged over a window of at least - // MIN_FRAMES_DECODED_WINDOW_MILLIS - private void waitUntilSubscriberFramesDecodedIncrease(OpenViduTestappUser user, WebElement videoElement) { - final long initialFramesDecoded = this.getSubscriberVideoFramesDecoded(user, videoElement); - final long initialFramesReceived = this.getSubscriberVideoFramesReceived(user, videoElement); - final long windowStart = System.currentTimeMillis(); - // Last sample taken by the loop, only to report it if the wait times out - final java.util.concurrent.atomic.AtomicLong lastFramesDecoded = new java.util.concurrent.atomic.AtomicLong(); - final java.util.concurrent.atomic.AtomicLong lastFramesReceived = new java.util.concurrent.atomic.AtomicLong(); - final java.util.concurrent.atomic.AtomicLong lastWindowMillis = new java.util.concurrent.atomic.AtomicLong(); - this.waitUntilAux(user, videoElement, () -> { - // Both counters must come from the very same dialog update: sampling - // them one by one would double the cost of every iteration - JsonObject layer = this.getSubscriberVideoLayer(user, videoElement); - long framesDecoded = this.getLayerCounter(layer, "framesDecoded"); - long framesReceived = this.getLayerCounter(layer, "framesReceived"); - if (framesDecoded < 0 || framesReceived < 0) { - return false; - } - long windowMillis = System.currentTimeMillis() - windowStart; - lastFramesDecoded.set(framesDecoded - initialFramesDecoded); - lastFramesReceived.set(framesReceived - initialFramesReceived); - lastWindowMillis.set(windowMillis); - // The window keeps growing while waiting, so a video that decodes a - // frame every now and then falls further behind the required rate - // instead of eventually satisfying it - return windowMillis >= MIN_FRAMES_DECODED_WINDOW_MILLIS - && lastFramesDecoded.get() * 1000 >= MIN_FRAMES_DECODED_FPS * windowMillis; - }, () -> { - long framesDecoded = lastFramesDecoded.get(); - long framesReceived = lastFramesReceived.get(); - long windowMillis = lastWindowMillis.get(); - // framesReceived counts the frames the depacketizer assembled, before - // handing them to the decoder. Comparing it against framesDecoded tells - // apart three failures that otherwise all look like "no video" - String diagnosis; - if (framesReceived <= 0) { - diagnosis = "The subscriber is not receiving assembled frames at all:" - + " the media is not reaching it"; - } else if (framesDecoded <= 0) { - diagnosis = "The subscriber IS receiving assembled frames (" + framesReceived - + ") but decoded none of them: the media that reaches it is undecodable" - + " (a Producer bound to the wrong codec, or a missing or wrong dependency" - + " descriptor)"; - } else { - diagnosis = "The subscriber received " + framesReceived + " assembled frame(s) and decoded " - + framesDecoded + " of them, but too slowly for a video that is actually playing"; - } - return "Timeout waiting for subscriber track to decode video at a sustained frame rate: only " - + framesDecoded + " frame(s) decoded in " + windowMillis + " ms (" - + String.format("%.2f", framesDecoded * 1000d / Math.max(1, windowMillis)) - + " fps), while at least " + MIN_FRAMES_DECODED_FPS - + " fps are required. Such a subscriber video is a frozen video. " + diagnosis; - }); - } - - private void waitUntilPublisherBytesSentIncrease(OpenViduTestappUser user, WebElement videoElement, String rid, - final long previousBytesSent) { - this.waitUntilAux(user, videoElement, () -> { - return this.getPublisherVideoLayerAttribute(user, videoElement, rid, "bytesSent") - .getAsLong() > previousBytesSent; - }, "Timeout waiting for publisher track to increase its bytesSent from " + previousBytesSent); - } - - private void waitUntilPublisherFramesEncodedIncrease(OpenViduTestappUser user, WebElement videoElement, String rid, - final long previousFramesEncoded) { - this.waitUntilAux(user, videoElement, () -> { - return this.getPublisherVideoLayerAttribute(user, videoElement, rid, "framesEncoded") - .getAsLong() > previousFramesEncoded; - }, "Timeout waiting for publisher track to increase its framesEncoded from " + previousFramesEncoded); - } - - private void waitUntilPublisherLayerActive(OpenViduTestappUser user, final WebElement publisherVideo, - final String rid, final boolean active) { - this.waitUntilAux(user, publisherVideo, () -> { - boolean currentlyActive = this.getPublisherVideoLayerAttribute(user, publisherVideo, rid, "active") - .getAsBoolean(); - if (active) { - JsonElement frameWidth = this.getPublisherVideoLayerAttribute(user, publisherVideo, rid, "frameWidth"); - return currentlyActive && frameWidth != null; - } else { - return !currentlyActive; - } - }, "Timeout waiting for video track layer to be " + (active ? "active" : "inactive")); - } - - private void waitUntilAux(OpenViduTestappUser user, WebElement videoElement, - Callable breakFromLoopFunction, String errMsg) { - this.waitUntilAux(user, videoElement, breakFromLoopFunction, () -> errMsg); - } - - // Same as above, but building the error message only if the wait times out, so - // that it can report the values actually observed by the last iteration - private void waitUntilAux(OpenViduTestappUser user, WebElement videoElement, - Callable breakFromLoopFunction, java.util.function.Supplier errMsg) { - try { - final long intervalWait = 250; - final long deadline = System.currentTimeMillis() + WAIT_UNTIL_MAX_MILLIS; - boolean breakFromLoop = false; - while (!breakFromLoop && System.currentTimeMillis() < deadline) { - try { - breakFromLoop = breakFromLoopFunction.call(); - } catch (Exception e1) { - e1.printStackTrace(); - } - if (breakFromLoop) { - break; - } else { - try { - Thread.sleep(intervalWait); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - if (!breakFromLoop) { - Assertions.fail(errMsg.get()); - } - } finally { - // Best-effort close of the info dialog - try { - if (!user.getDriver().findElements(By.cssSelector("#close-dialog-btn")).isEmpty()) { - this.waitForBackdropAndClick(user, "#close-dialog-btn"); - Thread.sleep(500); - } - } catch (Exception e) { - log.warn("Best-effort info-dialog close failed (ignored): {}", e.getMessage()); - } - } - } - - private void openInfoDialog(OpenViduTestappUser user, WebElement video) { - String videoId = video.getDomProperty("id"); - // Open the track info dialog if required - boolean dialogWasOpened; - if (!user.getDriver().findElements(By.cssSelector("app-info-dialog")).isEmpty()) { - // Dialog already opened - if (!user.getDriver().findElement(By.cssSelector("#subtitle")).getText().equals(videoId)) { - // Wrong dialog - this.waitForBackdropAndClick(user, "#close-dialog-btn"); - this.waitForBackdropAndClick(user, "#" + videoId + " ~ .bottom-div .video-track-info"); - dialogWasOpened = true; - } else { - dialogWasOpened = false; - } - } else { - // Dialog is not opened - this.waitForBackdropAndClick(user, "#" + videoId + " ~ .bottom-div .video-track-info"); - dialogWasOpened = true; - } - if (dialogWasOpened) { - try { - Thread.sleep(300); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - private void addPublisherSubscriber(OpenViduTestappUser user, boolean hasAudio, boolean hasVideo) - throws InterruptedException { - this.addPublisher(user, true, true, true, true, hasAudio, hasVideo, null, null, null); - } - - private void addOnlyPublisherVideo(OpenViduTestappUser user, boolean simulcast, boolean dynacast, boolean hd) - throws InterruptedException { - if (hd) { - this.addPublisher(user, false, simulcast, dynacast, false, false, true, 1920, 1080, null); - } else { - this.addPublisher(user, false, simulcast, dynacast, false, false, true, null, null, null); - } - } - - private void addOnlyPublisherVideo(OpenViduTestappUser user, boolean simulcast, boolean dynacast, boolean hd, - String scalabilityMode) - throws InterruptedException { - if (hd) { - this.addPublisher(user, false, simulcast, dynacast, false, false, true, 1920, 1080, scalabilityMode); - } else { - this.addPublisher(user, false, simulcast, dynacast, false, false, true, null, null, null); - } - } - - private void addOnlyPublisherAudio(OpenViduTestappUser user) throws InterruptedException { - this.addPublisher(user, false, false, false, false, true, false, null, null, null); - } - - private void addPublisher(OpenViduTestappUser user, boolean isSubscriber, boolean simulcast, boolean dynacast, - boolean adaptiveStream, boolean hasAudio, boolean hasVideo, Integer width, Integer height, - String scalabilityMode) throws InterruptedException { - if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { - user.getDriver().findElement(By.id("close-dialog-btn")).click(); - Thread.sleep(300); - } - final int previousInstances = user.getDriver().findElements(By.cssSelector("app-openvidu-instance")).size(); - user.getDriver().findElement(By.id("add-user-btn")).click(); - // The new instance is rendered asynchronously: counting the instances right - // after the click can still see only the previous ones, and then every - // "#openvidu-instance-" selector built from that count is off by one - // (with a single instance it even becomes "#openvidu-instance--1") - user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.cssSelector("app-openvidu-instance"), - previousInstances + 1)); - int numberOfUser = previousInstances; - if (!isSubscriber) { - user.getDriver().findElement(By.cssSelector("#openvidu-instance-" + numberOfUser + " .subscriber-checkbox")) - .click(); - } - this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); - Thread.sleep(300); - if (!hasAudio) { - user.getDriver().findElement(By.id("audio-capture-false")).click(); - } else { - user.getDriver().findElement(By.id("audio-capture-true")).click(); - } - if (!hasVideo) { - user.getDriver().findElement(By.id("video-capture-false")).click(); - } else { - user.getDriver().findElement(By.id("video-capture-true")).click(); - if (width != null || height != null || scalabilityMode != null) { - this.setPublisherCustomVideoProperties(user, width, height, scalabilityMode); - } - } - if (!simulcast) { - user.getDriver().findElement(By.id("trackPublish-simulcast")).click(); - } - if (!dynacast) { - user.getDriver().findElement(By.id("room-dynacast")).click(); - } - if (!adaptiveStream) { - user.getDriver().findElement(By.id("room-adaptiveStream")).click(); - } - user.getDriver().findElement(By.id("close-dialog-btn")).click(); - Thread.sleep(300); - } - - private void addSubscriber(OpenViduTestappUser user, boolean adaptiveStream) throws InterruptedException { - if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { - user.getDriver().findElement(By.id("close-dialog-btn")).click(); - Thread.sleep(300); - } - final int previousInstances = user.getDriver().findElements(By.cssSelector("app-openvidu-instance")).size(); - user.getDriver().findElement(By.id("add-user-btn")).click(); - // The new instance is rendered asynchronously: counting the instances right - // after the click can still see only the previous ones, and then every - // "#openvidu-instance-" selector built from that count is off by one - // (with a single instance it even becomes "#openvidu-instance--1") - user.getWaiter().until(ExpectedConditions.numberOfElementsToBe(By.cssSelector("app-openvidu-instance"), - previousInstances + 1)); - int numberOfUser = previousInstances; - user.getDriver().findElement(By.cssSelector("#openvidu-instance-" + numberOfUser + " .publisher-checkbox")) - .click(); - if (!adaptiveStream) { - this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); - this.waitForBackdropAndClick(user, "#room-adaptiveStream"); - user.getDriver().findElement(By.id("close-dialog-btn")).click(); - Thread.sleep(300); - } - } - - private void createIngress(OpenViduTestappUser user, String preset, String codec, boolean simulcast, String urlType, - String urlUri) throws InterruptedException { - if (!user.getDriver().findElements(By.id("close-dialog-btn")).isEmpty()) { - this.waitForBackdropAndClick(user, "#close-dialog-btn"); - Thread.sleep(300); - } - user.getDriver().findElement(By.xpath("//button[contains(@title,'Room API')]")).click(); - if (preset != null) { - this.waitForBackdropAndClick(user, "#ingress-preset-select"); - this.waitForBackdropAndClick(user, "#mat-option-" + preset.toUpperCase()); - } else { - if (!simulcast) { - this.waitForBackdropAndClick(user, "#ingress-simulcast"); - Thread.sleep(300); - } - this.waitForBackdropAndClick(user, "#ingress-video-codec-select"); - this.waitForBackdropAndClick(user, "#mat-option-" + codec.toUpperCase()); - } - if (urlType != null) { - this.waitForBackdropAndClick(user, "#ingress-url-type-select"); - this.waitForBackdropAndClick(user, "#mat-option-" + urlType.toUpperCase()); - } - if (urlUri != null) { - user.getDriver().findElement(By.cssSelector("#ingress-url-uri-field")).sendKeys(urlUri); - Thread.sleep(300); - } - this.waitForBackdropAndClick(user, "#create-ingress-api-btn"); - this.waitForBackdropAndClick(user, "#close-dialog-btn"); - Thread.sleep(300); - } - - private void setPublisherSimulcastLayersAndResolution(OpenViduTestappUser user, int numberOfUser, - String simulcastLayerName, Integer width, Integer height) throws InterruptedException { - this.waitForBackdropAndClick(user, "#room-options-btn-" + numberOfUser); - Thread.sleep(300); - this.setPublisherCustomVideoProperties(user, width, height, null); - user.getDriver().findElement(By.id("trackPublish-videoSimulcastLayers")).click(); - this.waitForBackdropAndClick(user, "#mat-option-" + simulcastLayerName); - new org.openqa.selenium.interactions.Actions(user.getDriver()) - .sendKeys(org.openqa.selenium.Keys.ESCAPE).perform(); - Thread.sleep(300); - this.waitForBackdropAndClick(user, "#close-dialog-btn"); - Thread.sleep(300); - } - - private void setPublisherCustomVideoProperties(OpenViduTestappUser user, Integer width, Integer height, - String scalabilityMode) { - user.getDriver().findElement(By.id("video-capture-custom")).click(); - if (width != null) { - WebElement trackWidth = user.getDriver().findElement(By.id("resolution-video-capture-options-width")); - trackWidth.clear(); - trackWidth.sendKeys(width.toString()); - } - if (height != null) { - WebElement trackHeight = user.getDriver().findElement(By.id("resolution-video-capture-options-height")); - trackHeight.clear(); - trackHeight.sendKeys(height.toString()); - } - if (scalabilityMode != null) { - user.getDriver().findElement(By.id("trackPublish-scalabilityMode")).click(); - this.waitForBackdropAndClick(user, ".mode-" + scalabilityMode); - } - } - - /** - * Waits for any Material Design backdrop overlays to disappear and then clicks - * the element. This prevents ElementClickInterceptedException caused by overlay - * backdrops. - */ - private void waitForBackdropAndClick(OpenViduTestappUser user, String cssSelector) { - final long startTime = System.currentTimeMillis(); - final long timeoutMillis = 10000; // 10 seconds total timeout - final long retryIntervalMillis = 500; // 500ms between retries - - WebElement element = null; - - while (System.currentTimeMillis() - startTime < timeoutMillis) { - try { - // Try to find and click the element immediately - element = user.getDriver().findElement(By.cssSelector(cssSelector)); - if (element.isDisplayed() && element.isEnabled()) { - element.click(); - return; // Success! Exit the method - } - } catch (org.openqa.selenium.ElementClickInterceptedException e) { - // Element is being intercepted by overlay, continue retrying - } catch (org.openqa.selenium.NoSuchElementException e) { - // Element not found, wait a bit and retry - } catch (org.openqa.selenium.StaleElementReferenceException e) { - // Element reference is stale, retry with fresh element - } catch (Exception e) { - // Any other exception, continue retrying - } - - // Wait before next retry - try { - Thread.sleep(retryIntervalMillis); - } catch (InterruptedException e) { - // Print screenshot - String screenshot = "data:image/png;base64," - + ((TakesScreenshot) user.getDriver()).getScreenshotAs(BASE64); - System.out.println("INTERRUPTED EXCEPTION WHILE WAITING FOR ELEMENT TO BE CLICKABLE: " + cssSelector); - System.out.println(screenshot); - Thread.currentThread().interrupt(); - throw new RuntimeException("Thread interrupted while waiting for backdrop to clear", e); - } - } - - String screenshot = "data:image/png;base64," + ((TakesScreenshot) user.getDriver()).getScreenshotAs(BASE64); - System.out.println("TIMEOUT WAITING FOR ELEMENT TO BE CLICKABLE (): " + cssSelector); - System.out.println(screenshot); - - // If we get here, we've timed out - throw new RuntimeException("Timeout waiting for element '" + cssSelector - + "' to be clickable without backdrop interference after " + timeoutMillis + "ms"); - } - - public boolean assertAllElementsHaveTracks(OpenViduTestappUser user, String selector, boolean hasAudio, - boolean hasVideo) { - org.openqa.selenium.JavascriptExecutor js = (org.openqa.selenium.JavascriptExecutor) user.getDriver(); - String script = "var elements = document.querySelectorAll(arguments[0]);" + - "for (var i = 0; i < elements.length; i++) {" + - " var el = elements[i];" + - " if (!el.srcObject) return false;" + - " if (arguments[1] && el.srcObject.getAudioTracks().length === 0) return false;" + - " if (!arguments[1] && el.srcObject.getAudioTracks().length > 0) return false;" + - " if (arguments[2] && el.srcObject.getVideoTracks().length === 0) return false;" + - " if (!arguments[2] && el.srcObject.getVideoTracks().length > 0) return false;" + - "}" + - "return true;"; - return (Boolean) js.executeScript(script, selector, hasAudio, hasVideo); - } - - public void changeElementSize(OpenViduTestappUser user, org.openqa.selenium.WebElement element, int width, - int height) { - org.openqa.selenium.JavascriptExecutor js = (org.openqa.selenium.JavascriptExecutor) user.getDriver(); - js.executeScript( - "arguments[0].style.width = '" + width + "px'; arguments[0].style.height = '" + height + "px';", - element); - } - }