diff --git a/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/NetworkConditioner.java b/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/NetworkConditioner.java new file mode 100644 index 000000000..0c7d70b98 --- /dev/null +++ b/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/NetworkConditioner.java @@ -0,0 +1,330 @@ +package io.openvidu.test.e2e; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.openvidu.test.browsers.utils.CommandLineExecutor; + +public class NetworkConditioner { + + private static final Logger log = LoggerFactory.getLogger(NetworkConditioner.class); + private static final CommandLineExecutor commandLine = new CommandLineExecutor(); + + private static final String PUMBA_IMAGE = "gaiaadm/pumba"; + private static final String NETTOOLS_IMAGE = "ghcr.io/alexei-led/pumba-alpine-nettools:latest"; + private static final String DOCKER_SOCK = "/var/run/docker.sock"; + + // Upper bound on expanded ports: Pumba creates one tc filter / iptables rule + // per port, so a huge range would be slow and fragile. The SFU RTC range (e.g. + // 7900-7999) is 100 ports. This value covers it. + private static final int MAX_EXPANDED_PORTS = 1024; + + // Name of the currently-running Pumba container to clear it. + private static String currentPumbaContainerName; + + // Container that currently has an OUTBOUND blackout rule (a single iptables + // OUTPUT DROP added + // directly, not via Pumba). Tracked so clear() can flush it. See + // blackoutOutbound(). + private static String blackoutContainer; + + public enum Direction { + OUTBOUND, INBOUND + } + + /** + * L4 protocol to scope the drop to. Only honored for {@link Direction#INBOUND} + */ + public enum Protocol { + UDP, TCP + } + + public static void pullImages() { + log.info("Pulling Pumba images {} and {}", PUMBA_IMAGE, NETTOOLS_IMAGE); + commandLine.executeCommand("docker pull " + PUMBA_IMAGE, 180); + commandLine.executeCommand("docker pull " + NETTOOLS_IMAGE, 180); + } + + public static void applyLossToOutboundPackets(String targetContainer, String remotePorts, int lossPercent, + int durationSec) { + log.info("Dropping " + lossPercent + "% of packets leaving container " + targetContainer + + " with destination port " + remotePorts + " during " + durationSec + " seconds"); + applyLoss(targetContainer, NetworkConditioner.Direction.OUTBOUND, null, remotePorts, lossPercent, durationSec); + } + + public static void applyLossToInboundPackets(String targetContainer, Protocol protocol, String remotePorts, + int lossPercent, int durationSec) { + log.info("Dropping " + lossPercent + "% of packets entering container " + targetContainer + + " with origin port " + remotePorts + " during " + durationSec + " seconds"); + applyLoss(targetContainer, NetworkConditioner.Direction.INBOUND, protocol, remotePorts, lossPercent, + durationSec); + } + + /** + * Update the OUTBOUND packet-loss percentage in place (no Pumba restart). + * {@code tc qdisc change} rewrites only the netem loss parameter of the qdisc + * Pumba installed for the port-scoped egress filter, preserving the queue and + * the port filters. Requires a prior {@link #applyLossToOutboundPackets} with a + * {@code durationSec} long enough to span the whole ramp. + */ + public static void updateOutboundLossPercent(String targetContainer, int lossPercent) { + log.info("Updating OUTBOUND packet loss on container {} to {}% in place (tc qdisc change, no Pumba restart)", + targetContainer, lossPercent); + // --entrypoint tc is REQUIRED: the nettools image's default entrypoint is `tail + // -f /dev/null`, so without overriding it the tc args would be handed to tail. + // The netem qdisc MUST be named by its handle (30:) — `tc qdisc change` looks + // it up by handle, so `parent 1:3` alone fails with "Failed to find specified + // qdisc". Pumba adds it as `parent 1:3 handle 30:` (root prio 1: -> netem on + // band 1:3), which we mirror here. + String cmd = "docker run --rm --network container:" + targetContainer + + " --cap-add NET_ADMIN --entrypoint tc " + NETTOOLS_IMAGE + + " qdisc change dev eth0 parent 1:3 handle 30: netem loss " + lossPercent + "% 2>&1"; + String out = commandLine.executeCommand(cmd, 30); + log.info("tc qdisc change result: {}", out); + } + + /** + * Update the INBOUND packet-loss percentage in place + */ + public static void updateInboundLossPercent(String targetContainer, Protocol protocol, String remotePorts, + int lossPercent) { + final String ports = expandPorts(remotePorts); + if (ports == null) { + throw new IllegalArgumentException( + "updateInboundLossPercent needs the same remotePorts passed to applyLossToInboundPackets"); + } + final String proto = protocol.name().toLowerCase(Locale.US); + final String probability = String.format(Locale.US, "%.2f", Math.max(0.0, Math.min(1.0, lossPercent / 100.0))); + log.info("Updating INBOUND packet loss on container {} to {}% in place (iptables -R, no Pumba restart)", + targetContainer, lossPercent); + + // 1) Read the live INPUT rules (spec form) to find each port's DROP rule index + String listCmd = "docker run --rm --network container:" + targetContainer + + " --cap-add NET_ADMIN --entrypoint iptables " + NETTOOLS_IMAGE + " -S INPUT 2>&1"; + String[] rules = commandLine.executeCommand(listCmd, 30).split("-A INPUT"); + + // 2) Build an atomic `iptables -R` per target port (rules[0] is the "-P INPUT + // ..." policy). + StringBuilder script = new StringBuilder(); + for (String port : ports.split(",")) { + int ruleNumber = -1; + for (int i = 1; i < rules.length; i++) { + if (rules[i].contains("--sport " + port + " ") && rules[i].contains("statistic") + && rules[i].contains("DROP")) { + ruleNumber = i; + break; + } + } + if (ruleNumber < 0) { + log.warn("No INPUT DROP rule found for source port {} on container {}; skipping", port, + targetContainer); + continue; + } + if (script.length() > 0) { + script.append("; "); + } + script.append("iptables -R INPUT ").append(ruleNumber).append(" -i eth0 -p ").append(proto) + .append(" --sport ").append(port).append(" -m statistic --mode random --probability ") + .append(probability).append(" -j DROP"); + } + if (script.length() == 0) { + log.warn("updateInboundLossPercent: no matching INPUT rules found on container {}; nothing updated", + targetContainer); + return; + } + + // 3) Apply all replacements in one sidecar + String replaceCmd = "docker run --rm --network container:" + targetContainer + + " --cap-add NET_ADMIN --entrypoint sh " + NETTOOLS_IMAGE + " -c \"" + script + "\" 2>&1"; + String out = commandLine.executeCommand(replaceCmd, 30); + log.info("iptables -R result: {}", out); + } + + /** + * Drop a percentage of packets matching {@code direction} + {@code protocol} + + * {@code remotePorts}. + * + * @param targetContainer docker name/ID of the impaired container. Should never + * be --network=host + * @param direction {@link Direction#OUTBOUND} (egress, via netem) or + * {@link Direction#INBOUND} (ingress, via iptables) + * @param protocol {@link Protocol#UDP}/{@link Protocol#TCP}; honored for + * INBOUND, logged-and-ignored for OUTBOUND (netem cannot + * filter protocol) + * @param remotePorts the SFU/remote port(s): a single port ("7780"), a + * comma list ("7880,7881") or a range + * ("7900-7999"/"7900:7999"); expanded to a comma list + * (Pumba accepts no range syntax). null/blank = no port + * filter (whole interface). + * @param lossPercent packet loss percentage (0-100); for INBOUND this maps + * to a 0.0-1.0 per-packet probability + * @param durationSec how long Pumba keeps the impairment before + * auto-reverting + */ + private static void applyLoss(String targetContainer, Direction direction, Protocol protocol, String remotePorts, + int lossPercent, int durationSec) { + final String ports = expandPorts(remotePorts); + final String cmd; + if (direction == Direction.OUTBOUND) { + if (protocol != null) { + log.warn("Pumba netem (egress) cannot filter by L4 protocol; ignoring protocol={} for OUTBOUND loss " + + "(the SFU RTC media ports are UDP-only, so this is equivalent to a UDP filter).", protocol); + } + StringBuilder opts = new StringBuilder("--duration ").append(durationSec).append("s --interface eth0") + .append(" --tc-image ").append(NETTOOLS_IMAGE); + if (ports != null) { + // remotePorts = SFU ports = DESTINATION of the client's egress packets => + // --ingress-port (dport) + opts.append(" --ingress-port ").append(ports); + } + cmd = pumbaRun() + " netem " + opts + " loss --percent " + lossPercent + " " + targetContainer; + } else { + double probability = Math.max(0.0, Math.min(1.0, lossPercent / 100.0)); + StringBuilder opts = new StringBuilder("--duration ").append(durationSec).append("s --interface eth0") + .append(" --iptables-image ").append(NETTOOLS_IMAGE); + if (protocol != null) { + opts.append(" --protocol ").append(protocol.name().toLowerCase(Locale.US)); + } + if (ports != null) { + // remotePorts = SFU ports = SOURCE of the client's ingress packets => + // --src-port (sport). + // Note: iptables only allows a port filter together with a tcp/udp protocol. + if (protocol == null) { + throw new IllegalArgumentException( + "INBOUND loss with a port filter requires a protocol (Pumba iptables --src-port needs -p tcp/udp)"); + } + opts.append(" --src-port ").append(ports); + } + cmd = pumbaRun() + " iptables " + opts + " loss --mode random --probability " + + String.format(Locale.US, "%.2f", probability) + " " + targetContainer; + } + runPumba(cmd); + } + + /** + * Total OUTBOUND blackout: 100% packet loss across an ENTIRE SFU media port + * range, superseding any in-place single-port impairment. One native range rule + * (iptables matches {@code low:high} directly) reliably blocks all media, + * unlike a flaky 100-filter netem + * + * Stops the running Pumba first (avoids conflicting root qdiscs), then installs + * a SINGLE iptables OUTPUT DROP rule over the whole {@code mediaPortRange} + * (e.g. "7900-7999"). + */ + public static void blackoutOutbound(String targetContainer, String mediaPortRange, int durationSec) { + clear(); + final String iptablesRange = mediaPortRange.replace('-', ':'); // iptables ranges are low:high + log.info("Total OUTBOUND blackout (100% loss) on container {} across SFU media port range {} " + + "(single iptables OUTPUT DROP)", targetContainer, mediaPortRange); + String cmd = "docker run --rm --network container:" + targetContainer + + " --cap-add NET_ADMIN --entrypoint iptables " + NETTOOLS_IMAGE + + " -A OUTPUT -o eth0 -p udp --dport " + iptablesRange + " -j DROP 2>&1"; + String out = commandLine.executeCommand(cmd, 30); + log.info("blackout iptables -A OUTPUT result: {}", out); + blackoutContainer = targetContainer; + } + + /** + * Add delay + jitter to the target container's. OUTBOUND only: Pumba can only + * delay with netem/tc, which is egress-only. + */ + public static void applyDelay(String targetContainer, int delayMs, int jitterMs, int durationSec, + String remotePorts) { + final String ports = expandPorts(remotePorts); + StringBuilder opts = new StringBuilder("--duration ").append(durationSec).append("s --interface eth0") + .append(" --tc-image ").append(NETTOOLS_IMAGE); + if (ports != null) { + opts.append(" --ingress-port ").append(ports); + } + String cmd = pumbaRun() + " netem " + opts + " delay --time " + delayMs + " --jitter " + jitterMs + " " + + targetContainer; + runPumba(cmd); + } + + /** + * Stop the current Pumba container (SIGTERM) so it reverts the netem qdisc / + * iptables rule immediately. + */ + public static void clear() { + if (currentPumbaContainerName != null) { + log.info("Clearing network impairment (stopping Pumba container {})", currentPumbaContainerName); + commandLine.executeCommand("docker stop -t 5 " + currentPumbaContainerName, 30); + currentPumbaContainerName = null; + } + if (blackoutContainer != null) { + log.info("Clearing OUTBOUND blackout (flushing iptables OUTPUT) on container {}", blackoutContainer); + // Flush the OUTPUT chain: the blackout DROP is the only rule we ever add there + String cmd = "docker run --rm --network container:" + blackoutContainer + + " --cap-add NET_ADMIN --entrypoint iptables " + NETTOOLS_IMAGE + " -F OUTPUT 2>&1"; + commandLine.executeCommand(cmd, 30); + blackoutContainer = null; + } + } + + /** + * Expand a port spec into the comma-separated single-port list Pumba expects. + * Accepts single ports, comma lists and {@code from-to} / {@code from:to} + * ranges. Returns {@code null} for a blank spec (no port filter). + */ + static String expandPorts(String spec) { + if (spec == null || spec.isBlank()) { + return null; + } + List out = new ArrayList<>(); + for (String rawToken : spec.split(",")) { + String token = rawToken.trim(); + if (token.isEmpty()) { + continue; + } + String sep = token.contains("-") ? "-" : (token.contains(":") ? ":" : null); + if (sep != null) { + String[] parts = token.split(Pattern.quote(sep)); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid port range: " + token); + } + int from = parsePort(parts[0].trim()); + int to = parsePort(parts[1].trim()); + if (from > to) { + int tmp = from; + from = to; + to = tmp; + } + for (int p = from; p <= to; p++) { + out.add(String.valueOf(p)); + } + } else { + out.add(String.valueOf(parsePort(token))); + } + if (out.size() > MAX_EXPANDED_PORTS) { + throw new IllegalArgumentException("Port spec '" + spec + "' expands to more than " + MAX_EXPANDED_PORTS + + " ports; Pumba creates one rule/filter per port. Narrow the range."); + } + } + return out.isEmpty() ? null : String.join(",", out); + } + + private static int parsePort(String s) { + int port = Integer.parseInt(s); + if (port < 0 || port > 65535) { + throw new IllegalArgumentException("Port out of range (0-65535): " + port); + } + return port; + } + + private static String pumbaRun() { + currentPumbaContainerName = "pumba-netem-" + System.currentTimeMillis(); + return "docker run -d --name " + currentPumbaContainerName + " --rm -v " + DOCKER_SOCK + ":" + DOCKER_SOCK + " " + + PUMBA_IMAGE; + } + + private static void runPumba(String cmd) { + log.info("Applying network impairment via Pumba: {}", cmd); + String out = commandLine.executeCommand(cmd, 60); + log.info("Pumba started: {}", out); + } +} diff --git a/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/OpenViduTestE2e.java b/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/OpenViduTestE2e.java index 4dbb65190..c1a0e97d3 100644 --- a/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/OpenViduTestE2e.java +++ b/openvidu-test-e2e/src/main/java/io/openvidu/test/e2e/OpenViduTestE2e.java @@ -18,6 +18,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import javax.net.ssl.SSLContext; @@ -31,6 +32,7 @@ import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.WaitStrategy; import org.testcontainers.utility.DockerImageName; @@ -71,7 +73,8 @@ public class OpenViduTestE2e { put("VP9", Triple.of("libvpx-vp9", "", "VP9")); put("MPEG-4", Triple.of("mpeg4", "", "MPEG-4")); put("M-JPEG", Triple.of("mjpeg", "-force_duplicated_matrix:v 1 -huffman:v 0", "M-JPEG")); - // put("AV1", Triple.of("libaom-av1", "", "AV1")); // NOT SUPPORTED BY THE RTSP SERVER + // put("AV1", Triple.of("libaom-av1", "", "AV1")); // NOT SUPPORTED BY THE RTSP + // SERVER // (maybe gstreamer?) // put("H265", Triple.of("libx265", "", "H265")); // NOT SUPPORTED BY INGRESS } @@ -119,6 +122,19 @@ public class OpenViduTestE2e { protected Collection browserUsers = ConcurrentHashMap.newKeySet(); protected static Collection> containers = ConcurrentHashMap.newKeySet(); + protected final int CONNECTION_QUALITY_OBSERVATION_MS = 22000; + protected static final int CONNECTION_QUALITY_STABLE_MS = 18000; + + // Connection Quality Group 2: tracking for the isolated "chromeNetwork" + // browsers. Several can run at once (e.g. an impaired peer + a clean peer in + // the same test), so each gets a UNIQUE container name — keyed by its + // BrowserUser so a test can resolve the right one for Pumba via + // getNetemContainerName(user) — and its OWN docker network. All networks are + // closed in dispose(). + protected Map netemContainerNames = new ConcurrentHashMap<>(); + protected Collection netemNetworks = ConcurrentHashMap.newKeySet(); + private final AtomicInteger netemContainerCounter = new AtomicInteger(); + protected static RoomServiceClient LK; protected static IngressServiceClient LK_INGRESS; @@ -151,6 +167,66 @@ public class OpenViduTestE2e { return chrome; } + // Connection Quality Group 2: a Chrome container on its OWN isolated network + // namespace so Pumba + // can apply netem to ONLY this client's traffic (NOT host networking, which + // would impair the + // whole host, SFU included). It joins a DEDICATED, uniquely-named docker + // network created just + // for this test (not the shared default bridge), so the impairment never + // touches any other + // docker network. Each network is tracked in `netemNetworks` and closed in + // dispose(). + // + // The client does NOT need to share the SFU's docker network. Signalling + // reaches the secure + // wss://:7443 URL (host-published by caddy-proxy), and WebRTC + // media reaches the + // SFU's advertised host LAN IP candidate (e.g. 10.10.1.203:7900-7999/udp), + // which the SFU + // publishes to the host. Any docker bridge can reach a host-published port via + // the host IP + + // DNAT — docker's inter-bridge isolation only blocks direct container↔container + // routing, not + // the published-port path — and the SFU being ICE-lite means the client just + // sends to that one + // candidate and gets replies back via UDP conntrack. So full network isolation + // works. + // + // The WebDriver port 4444 is published to a mapped host port for the + // RemoteWebDriver, and + // host.docker.internal (host-gateway) reaches the host-published testapp. + private GenericContainer chromeContainerNetem(String image, long shmSize, int maxBrowserSessions, + boolean headless, String containerName) { + Map map = new HashMap<>(); + map.put("SE_OPTS", "--port 4444"); + map.put("SE_EVENT_BUS_PUBLISH_PORT", "4542"); + map.put("SE_EVENT_BUS_SUBSCRIBE_PORT", "4543"); + if (headless) { + map.put("START_XVFB", "false"); + } + if (maxBrowserSessions > 1) { + map.put("SE_NODE_OVERRIDE_MAX_SESSIONS", "true"); + map.put("SE_NODE_MAX_SESSIONS", String.valueOf(maxBrowserSessions)); + } + Network netemNetwork = Network.builder() + .createNetworkCmdModifier(cmd -> cmd.withName(containerName + "-net")) + .build(); + this.netemNetworks.add(netemNetwork); + GenericContainer chrome = new GenericContainer<>(DockerImageName.parse(image)).withSharedMemorySize(shmSize) + .withFileSystemBind("/opt/openvidu", "/opt/openvidu").withEnv(map) + .withNetwork(netemNetwork) + .withExposedPorts(4444) + .withExtraHost("host.docker.internal", "host-gateway") + .withCreateContainerCmdModifier(cmd -> cmd.withName(containerName)) + .waitingFor(waitBrowser); + return chrome; + } + + protected String getNetemContainerName(BrowserUser browserUser) { + return this.netemContainerNames.get(browserUser); + } + private GenericContainer firefoxContainer(String image, long shmSize, int maxBrowserSessions, boolean headless) { Map map = new HashMap<>(); map.put("SE_OPTS", "--port 4445"); @@ -232,12 +308,14 @@ public class OpenViduTestE2e { // e.g. "[path live] stream is available and online, 2 tracks (H264, Opus)" if (videoCodec != null) { String expectedVideoCodecLogValue = FFMPEG_VIDEO_CODEC_NAMES.get(videoCodec).getRight(); - String regex = ".*\\[path " + RTSP_PATH + "\\] stream is available.*\\(.*(?i)(" + expectedVideoCodecLogValue + ").*\\).*"; + String regex = ".*\\[path " + RTSP_PATH + "\\] stream is available.*\\(.*(?i)(" + expectedVideoCodecLogValue + + ").*\\).*"; waitUntilLog(rtspServerContainer, regex, 15); } if (audioCodec != null) { String expectedValue = FFMPEG_AUDIO_CODEC_NAMES.get(audioCodec).getRight(); - String regex = ".*\\[path " + RTSP_PATH + "\\] stream is available.*\\(.*(?i)(" + expectedValue + ").*\\).*"; + String regex = ".*\\[path " + RTSP_PATH + "\\] stream is available.*\\(.*(?i)(" + expectedValue + + ").*\\).*"; waitUntilLog(rtspServerContainer, regex, 15); } @@ -435,77 +513,105 @@ public class OpenViduTestE2e { boolean headless = false; switch (browser) { - case "chrome": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, headless); - setupBrowserAux(BrowserNames.CHROME, container, false); - browserUser = new ChromeUser("TestUser", 50, headless); - break; - case "chromeTwoInstances": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 2, headless); - setupBrowserAux(BrowserNames.CHROME, container, false); - browserUser = new ChromeUser("TestUser", 50, headless); - break; - case "chromeAlternateScreenShare": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.CHROME, container, false); - browserUser = new ChromeUser("TestUser", 50, "OpenVidu TestApp"); - break; - case "chromeAlternateFakeVideo": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.CHROME, container, false); - path = Paths.get("/opt/openvidu/barcode.y4m"); - checkMediafilePath(path); - browserUser = new ChromeUser("TestUser", 50, path); - break; - case "chromeFakeAudio": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.CHROME, container, false); - path = new File("/opt/openvidu/test.wav").toPath(); - try { - checkMediafilePath(path); - } catch (Exception e) { + case "chrome": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, headless); + setupBrowserAux(BrowserNames.CHROME, container, false); + browserUser = new ChromeUser("TestUser", 50, headless); + break; + case "chromeTwoInstances": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 2, headless); + setupBrowserAux(BrowserNames.CHROME, container, false); + browserUser = new ChromeUser("TestUser", 50, headless); + break; + case "chromeNetwork": + // Bridged Chrome (in its own isolated Docker network) targetable by Pumba. A + // unique name + // (millis + counter) lets several run simultaneously; it is mapped to its + // BrowserUser below + // so a test can resolve it for Pumba via getNetemContainerName(user). + String netemContainerName = "openvidu-test-chrome-netem-" + System.currentTimeMillis() + "-" + + this.netemContainerCounter.incrementAndGet(); + container = chromeContainerNetem("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, + headless, + netemContainerName); + container.start(); + containers.add(container); + // Point the RemoteWebDriver at this container's mapped WebDriver port setting + // REMOTE_URL_CHROME. Restore the property afterwards + String previousRemoteUrlChrome = System.getProperty("REMOTE_URL_CHROME"); + System.setProperty("REMOTE_URL_CHROME", "http://localhost:" + container.getMappedPort(4444)); try { - FileUtils.copyURLToFile( - new URL("https://openvidu-loadtest-mediafiles.s3.amazonaws.com/interview.wav"), - new File("/opt/openvidu/test.wav"), 60000, 60000); - } catch (FileNotFoundException e2) { - e2.printStackTrace(); - System.err.println("exception on: downLoadFile() function: " + e.getMessage()); + browserUser = new ChromeUser("TestUser", 50, headless); + } finally { + if (previousRemoteUrlChrome == null) { + System.clearProperty("REMOTE_URL_CHROME"); + } else { + System.setProperty("REMOTE_URL_CHROME", previousRemoteUrlChrome); + } } - } - browserUser = new ChromeUser("TestUser", 50, null, path, headless); - break; - case "chromeDtxAudio": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.CHROME, container, false); - path = new File("/opt/openvidu/dtx_test_audio.wav").toPath(); - checkMediafilePath(path); - browserUser = new ChromeUser("TestUser", 50, null, path, headless); - break; - case "chromeVirtualBackgroundFakeVideo": - container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.CHROME, container, false); - path = Paths.get("/opt/openvidu/girl.mjpeg"); - checkMediafilePath(path); - browserUser = new ChromeUser("TestUser", 50, path, false); - break; - case "firefox": - container = firefoxContainer("selenium/standalone-firefox:" + FIREFOX_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.FIREFOX, container, false); - browserUser = new FirefoxUser("TestUser", 50, false, headless); - break; - case "firefoxDisabledOpenH264": - container = firefoxContainer("selenium/standalone-firefox:" + FIREFOX_VERSION, 2147483648L, 1, true); - setupBrowserAux(BrowserNames.FIREFOX, container, false); - browserUser = new FirefoxUser("TestUser", 50, true, headless); - break; - case "edge": - container = edgeContainer("selenium/standalone-edge:" + EDGE_VERSION, 2147483648L, 1, false); - setupBrowserAux(BrowserNames.EDGE, container, false); - browserUser = new EdgeUser("TestUser", 50, headless); - break; - default: - log.error("Browser {} not recognized", browser); + this.netemContainerNames.put(browserUser, netemContainerName); + break; + case "chromeAlternateScreenShare": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.CHROME, container, false); + browserUser = new ChromeUser("TestUser", 50, "OpenVidu TestApp"); + break; + case "chromeAlternateFakeVideo": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.CHROME, container, false); + path = Paths.get("/opt/openvidu/barcode.y4m"); + checkMediafilePath(path); + browserUser = new ChromeUser("TestUser", 50, path); + break; + case "chromeFakeAudio": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.CHROME, container, false); + path = new File("/opt/openvidu/test.wav").toPath(); + try { + checkMediafilePath(path); + } catch (Exception e) { + try { + FileUtils.copyURLToFile( + new URL("https://openvidu-loadtest-mediafiles.s3.amazonaws.com/interview.wav"), + new File("/opt/openvidu/test.wav"), 60000, 60000); + } catch (FileNotFoundException e2) { + e2.printStackTrace(); + System.err.println("exception on: downLoadFile() function: " + e.getMessage()); + } + } + browserUser = new ChromeUser("TestUser", 50, null, path, headless); + break; + case "chromeDtxAudio": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.CHROME, container, false); + path = new File("/opt/openvidu/dtx_test_audio.wav").toPath(); + checkMediafilePath(path); + browserUser = new ChromeUser("TestUser", 50, null, path, headless); + break; + case "chromeVirtualBackgroundFakeVideo": + container = chromeContainer("selenium/standalone-chrome:" + CHROME_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.CHROME, container, false); + path = Paths.get("/opt/openvidu/girl.mjpeg"); + checkMediafilePath(path); + browserUser = new ChromeUser("TestUser", 50, path, false); + break; + case "firefox": + container = firefoxContainer("selenium/standalone-firefox:" + FIREFOX_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.FIREFOX, container, false); + browserUser = new FirefoxUser("TestUser", 50, false, headless); + break; + case "firefoxDisabledOpenH264": + container = firefoxContainer("selenium/standalone-firefox:" + FIREFOX_VERSION, 2147483648L, 1, true); + setupBrowserAux(BrowserNames.FIREFOX, container, false); + browserUser = new FirefoxUser("TestUser", 50, true, headless); + break; + case "edge": + container = edgeContainer("selenium/standalone-edge:" + EDGE_VERSION, 2147483648L, 1, false); + setupBrowserAux(BrowserNames.EDGE, container, false); + browserUser = new EdgeUser("TestUser", 50, headless); + break; + default: + log.error("Browser {} not recognized", browser); } this.browserUsers.add(browserUser); @@ -536,20 +642,20 @@ public class OpenViduTestE2e { private static boolean isRemote(BrowserNames browser) { String remoteUrl = null; switch (browser) { - case CHROME: - remoteUrl = System.getProperty("REMOTE_URL_CHROME"); - break; - case FIREFOX: - remoteUrl = System.getProperty("REMOTE_URL_FIREFOX"); - break; - case OPERA: - remoteUrl = System.getProperty("REMOTE_URL_OPERA"); - break; - case EDGE: - remoteUrl = System.getProperty("REMOTE_URL_EDGE"); - break; - case ANDROID: - return true; + case CHROME: + remoteUrl = System.getProperty("REMOTE_URL_CHROME"); + break; + case FIREFOX: + remoteUrl = System.getProperty("REMOTE_URL_FIREFOX"); + break; + case OPERA: + remoteUrl = System.getProperty("REMOTE_URL_OPERA"); + break; + case EDGE: + remoteUrl = System.getProperty("REMOTE_URL_EDGE"); + break; + case ANDROID: + return true; } return remoteUrl != null; } @@ -576,6 +682,21 @@ public class OpenViduTestE2e { c.close(); it2.remove(); } + + // Remove the dedicated netem docker networks (created in chromeContainerNetem), + // now that their + // containers have been stopped. + Iterator itNet = this.netemNetworks.iterator(); + while (itNet.hasNext()) { + Network net = itNet.next(); + try { + net.close(); + } catch (Exception e) { + log.warn("Error closing netem docker network: {}", e.getMessage()); + } + itNet.remove(); + } + this.netemContainerNames.clear(); } protected void closeAllRooms(RoomServiceClient client) { diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java index bbaf0afc3..2c17633f5 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/AbstractOpenViduTestappE2eTest.java @@ -37,6 +37,10 @@ public class AbstractOpenViduTestappE2eTest extends OpenViduTestE2e { return testappUser; } + protected String getNetemContainerName(OpenViduTestappUser user) { + return this.getNetemContainerName(user.getBrowserUser()); + } + protected void gracefullyLeaveParticipants(OpenViduTestappUser user, int numberOfParticipants) throws Exception { int accumulatedDisconnected = 0; for (int j = 1; j <= numberOfParticipants; j++) { diff --git a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java index 03eadad1e..f54752f7d 100644 --- a/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java +++ b/openvidu-test-e2e/src/test/java/io/openvidu/test/e2e/OpenViduTestAppE2eTest.java @@ -25,9 +25,12 @@ import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -36,6 +39,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -46,6 +51,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.openqa.selenium.By; +import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.Keys; import org.openqa.selenium.TakesScreenshot; import org.openqa.selenium.WebElement; @@ -72,6 +78,7 @@ import io.minio.errors.XmlParserException; import io.minio.messages.Item; import livekit.LivekitIngress.IngressInfo; import livekit.LivekitIngress.IngressState; +import livekit.LivekitModels.ConnectionQuality; import static org.openqa.selenium.OutputType.BASE64; @@ -91,6 +98,13 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { checkFfmpegInstallation(); loadEnvironmentVariables(); setUpLiveKitClient(); + CompletableFuture.runAsync(() -> { + try { + NetworkConditioner.pullImages(); + } catch (Exception e) { + System.err.println("Download of images failed: " + e.getMessage()); + } + }); } @BeforeEach() @@ -381,6 +395,843 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest { gracefullyLeaveParticipants(user, 2); } + private List getConnectionQualityEventContents(OpenViduTestappUser user, int numberOfUser, + String participantName) { + String xpath = "//*[@id='openvidu-instance-" + numberOfUser + + "']//mat-expansion-panel[.//mat-expansion-panel-header[contains(@class,'connectionQualityChanged-" + + participantName + "')]]//div[contains(@class,'event-content')]"; + return user.getDriver().findElements(By.xpath(xpath)); + } + + // Latest connection-quality level currently reported for a participant, or null + // if none yet. + private ConnectionQuality latestConnectionQuality(OpenViduTestappUser user, int numberOfUser, + String participantName) { + List contents = getConnectionQualityEventContents(user, numberOfUser, participantName); + for (int i = contents.size() - 1; i >= 0; i--) { + String text = contents.get(i).getAttribute("textContent"); + if (text == null) { + continue; + } + text = text.toLowerCase().trim(); + if (text.contains("excellent")) { + return ConnectionQuality.EXCELLENT; + } + if (text.contains("good")) { + return ConnectionQuality.GOOD; + } + if (text.contains("poor")) { + return ConnectionQuality.POOR; + } + if (text.contains("lost")) { + return ConnectionQuality.LOST; + } + } + return null; + } + + // First loss% (ascending) at which the recorded quality equals `level`, or -1 + // if never reached. + private static int firstLossReaching(Map observed, ConnectionQuality level) { + for (Entry e : observed.entrySet()) { + if (e.getValue() == level) { + return e.getKey(); + } + } + return -1; + } + + // Render the ramp results (publisher vs subscriber-self quality per loss step) + // + the band transitions as a single log table. + private static String buildRampResultTable(Map publisher, + Map subscriber) { + StringBuilder sb = new StringBuilder("\nConnectionQuality ramp-up results\n"); + sb.append(" loss% | PunchbagUser (publisher) | RegularUser (subscriber, self)\n"); + sb.append(" ------+--------------------------+-------------------------------\n"); + ConnectionQuality prev = null; + for (Entry e : publisher.entrySet()) { + int pct = e.getKey(); + ConnectionQuality pub = e.getValue(); + ConnectionQuality sub = subscriber.get(pct); + String transition = (prev != null && pub != prev) ? " <- " + prev + " -> " + pub : ""; + sb.append(String.format(" %4d%% | %-24s | %-30s%s%n", pct, String.valueOf(pub), String.valueOf(sub), + transition)); + prev = pub; + } + sb.append(" transitions: EXCELLENT->GOOD @").append(firstLossReaching(publisher, ConnectionQuality.GOOD)) + .append("%, GOOD->POOR @").append(firstLossReaching(publisher, ConnectionQuality.POOR)) + .append("%, POOR->LOST @").append(firstLossReaching(publisher, ConnectionQuality.LOST)).append("%"); + return sb.toString(); + } + + private void assertConnectionQualityNeverDegraded(OpenViduTestappUser user, int numberOfUser, + String participantName) { + List contents = getConnectionQualityEventContents(user, numberOfUser, participantName); + Assertions.assertFalse(contents.isEmpty(), "Expected at least one connectionQualityChanged event for " + + participantName + " in openvidu-instance-" + numberOfUser); + String latest = null; + for (WebElement el : contents) { + String quality = el.getAttribute("textContent"); + if (quality == null) { + continue; + } + quality = quality.toLowerCase().trim(); + if (quality.isEmpty()) { + continue; + } + Assertions.assertFalse(quality.contains("poor") || quality.contains("lost"), + "Connection quality for " + participantName + " (openvidu-instance-" + numberOfUser + + ") must never be POOR or LOST under server-side-only conditions, but observed: '" + + quality + + "'"); + latest = quality; + } + Assertions.assertNotNull(latest, + "No connection quality value found for " + participantName + " in openvidu-instance-" + numberOfUser); + Assertions.assertTrue(latest.contains("excellent"), + "Connection quality for " + participantName + " (openvidu-instance-" + numberOfUser + + ") should settle on EXCELLENT, but the latest value was: '" + latest + "'"); + } + + @Test + @DisplayName("ConnectionQuality EXCELLENT with many published and subscribed tracks") + void connectionQualityManyTracksAlwaysExcellentTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("ConnectionQuality EXCELLENT with many published and subscribed tracks"); + + final int N = 3; + user.getDriver().findElement(By.id("auto-join-checkbox")).click(); + WebElement nInput = user.getDriver().findElement(By.id("one2many-input")); + nInput.clear(); + nInput.sendKeys(String.valueOf(N)); + user.getDriver().findElement(By.id("one2one-btn")).click(); + + // N participants, each publishing audio+video and subscribing to all others + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", N); + user.getEventManager().waitUntilEventReaches("localTrackPublished", "RoomEvent", 2 * N); + user.getEventManager().waitUntilEventReaches("connectionQualityChanged", "RoomEvent", N); + + // Let quality settle and give any (wrong) degradation time to surface + Thread.sleep(CONNECTION_QUALITY_OBSERVATION_MS); + + for (int i = 0; i < N; i++) { + assertConnectionQualityNeverDegraded(user, i, "TestParticipant" + i); + } + + gracefullyLeaveParticipants(user, N); + } + + @Test + @DisplayName("ConnectionQuality EXCELLENT when publisher pauses a track") + void connectionQualityPublisherPauseAlwaysExcellentTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("ConnectionQuality EXCELLENT when publisher pauses a track"); + + user.getDriver().findElement(By.id("auto-join-checkbox")).click(); + user.getDriver().findElement(By.id("one2one-btn")).click(); + + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 2); + user.getEventManager().waitUntilEventReaches("localTrackPublished", "RoomEvent", 4); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 4); + user.getEventManager().waitUntilEventReaches("connectionQualityChanged", "RoomEvent", 2); + + // Publisher (instance 0) pauses (mutes) its video track -> the scorer heals to + // EXCELLENT (mute -> cMaxScore), so quality must not degrade. + user.getDriver().findElement(By.cssSelector("#openvidu-instance-0 .mute-unmute-video")).sendKeys(Keys.ENTER); + user.getEventManager().waitUntilEventReaches("trackMuted", "RoomEvent", 2); + + Thread.sleep(CONNECTION_QUALITY_OBSERVATION_MS); + + assertConnectionQualityNeverDegraded(user, 0, "TestParticipant0"); + assertConnectionQualityNeverDegraded(user, 1, "TestParticipant1"); + + gracefullyLeaveParticipants(user, 2); + } + + @Test + @DisplayName("ConnectionQuality EXCELLENT when subscriber pauses a track") + void connectionQualitySubscriberPauseAlwaysExcellentTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("ConnectionQuality EXCELLENT when subscriber pauses a track"); + + user.getDriver().findElement(By.id("auto-join-checkbox")).click(); + user.getDriver().findElement(By.id("one2one-btn")).click(); + + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 2); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 4); + user.getEventManager().waitUntilEventReaches("connectionQualityChanged", "RoomEvent", 2); + + // Subscriber (instance 1) disables playback of the subscribed video track -> + // the + // DownTrack forwarder is muted and the downstream scorer heals to EXCELLENT. + user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 .toggle-video-enabled")).sendKeys(Keys.ENTER); + + Thread.sleep(CONNECTION_QUALITY_OBSERVATION_MS); + + assertConnectionQualityNeverDegraded(user, 0, "TestParticipant0"); + assertConnectionQualityNeverDegraded(user, 1, "TestParticipant1"); + + gracefullyLeaveParticipants(user, 2); + } + + @Test + @DisplayName("ConnectionQuality EXCELLENT when Dynacast pauses an unsubscribed track") + void connectionQualityDynacastPauseAlwaysExcellentTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("ConnectionQuality EXCELLENT when Dynacast pauses an unsubscribed track"); + + // Instance 0: video-only publisher with Dynacast + simulcast enabled. + this.addPublisher(user, false, true, true, false, false, true, null, null, null); + // Instance 1: subscriber only. + this.addSubscriber(user, true); + + user.getDriver().findElements(By.className("connect-btn")).forEach(el -> el.sendKeys(Keys.ENTER)); + + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 2); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 1); + user.getEventManager().waitUntilEventReaches("connectionQualityChanged", "RoomEvent", 1); + + // Subscriber unsubscribes from the only track -> the publisher's track becomes + // fully unwatched -> Dynacast pauses it -> the publisher stops sending and the + // mediasoup Producer score would decay to 0. Connection quality MUST stay + // EXCELLENT, NOT drop to POOR/LOST. + user.getDriver().findElement(By.cssSelector("#openvidu-instance-1 .toggle-video-subscribed")) + .sendKeys(Keys.ENTER); + + // Generous wait: Dynacast pause delay + producer inactivity (~1.5s) + EMA + + // room ticks + Thread.sleep(CONNECTION_QUALITY_OBSERVATION_MS + 10000); + + assertConnectionQualityNeverDegraded(user, 0, "TestParticipant0"); + + gracefullyLeaveParticipants(user, 2); + } + + @Test + @DisplayName("ConnectionQuality EXCELLENT when Adaptive Stream pauses a track") + void connectionQualityAdaptiveStreamPauseAlwaysExcellentTest() throws Exception { + + OpenViduTestappUser user = setupBrowserAndConnectToOpenViduTestapp("chrome"); + + log.info("ConnectionQuality EXCELLENT when Adaptive Stream pauses a track"); + + // one2one publishes audio+video with Adaptive Stream enabled (testapp default). + user.getDriver().findElement(By.id("auto-join-checkbox")).click(); + user.getDriver().findElement(By.id("one2one-btn")).click(); + + user.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 2); + user.getEventManager().waitUntilEventReaches("trackSubscribed", "RoomEvent", 4); + user.getEventManager().waitUntilEventReaches("connectionQualityChanged", "RoomEvent", 2); + + // Hide the remote video elements so Adaptive Stream pauses those subscriptions + // (Adaptive Stream pauses tracks whose