mirror of https://github.com/OpenVidu/openvidu.git
openvidu-test-e2e: added temporal networking diagnosis logs
parent
15b92f1afd
commit
b03080422a
|
|
@ -229,6 +229,38 @@ public class NetworkConditioner {
|
||||||
blackoutContainer = targetContainer;
|
blackoutContainer = targetContainer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dump connectivity diagnostics for a bridged (netem) container towards the SFU, to debug why the
|
||||||
|
* isolated browser can't establish (e.g. in CI). Everything is probed from INSIDE the container's
|
||||||
|
* own network namespace (via the nettools sidecar), so it reflects exactly what the browser sees:
|
||||||
|
* its own IPs/routes/DNS, DNS resolution of the LiveKit host, and TCP/ICMP reachability of the
|
||||||
|
* signaling port. Best-effort: each probe tolerates missing tools / failures (logged inline).
|
||||||
|
*/
|
||||||
|
public static void logConnectivityDiagnostics(String netemContainer, String host, int signalingPort) {
|
||||||
|
log.info("===== NETEM CONNECTIVITY DIAGNOSTICS (container={}, target={}:{}) =====", netemContainer, host,
|
||||||
|
signalingPort);
|
||||||
|
String script = String.join("; ", "echo '--- id/hostname ---'", "hostname 2>&1; id 2>&1",
|
||||||
|
"echo '--- container IPs (hostname -i) ---'", "hostname -i 2>&1",
|
||||||
|
"echo '--- all interfaces (ip -o addr) ---'", "ip -o addr 2>&1", "echo '--- routes (ip route) ---'",
|
||||||
|
"ip route 2>&1", "echo '--- /etc/resolv.conf ---'", "cat /etc/resolv.conf 2>&1",
|
||||||
|
"echo '--- DNS resolve " + host + " (nslookup) ---'", "nslookup " + host + " 2>&1",
|
||||||
|
"echo '--- ping " + host + " ---'", "ping -c3 -W2 " + host + " 2>&1",
|
||||||
|
"echo '--- TCP reach " + host + ":" + signalingPort + " (nc) ---'",
|
||||||
|
"nc -w4 -v " + host + " " + signalingPort + " </dev/null 2>&1",
|
||||||
|
"echo '--- HTTPS GET " + host + ":" + signalingPort + " (wget) ---'",
|
||||||
|
"wget -T6 -t1 --no-check-certificate -O /dev/null https://" + host + ":" + signalingPort + " 2>&1");
|
||||||
|
String cmd = "docker run --rm --network container:" + netemContainer + " --cap-add NET_ADMIN --entrypoint sh "
|
||||||
|
+ NETTOOLS_IMAGE + " -c \"" + script + "\" 2>&1";
|
||||||
|
String out;
|
||||||
|
try {
|
||||||
|
out = commandLine.executeCommand(cmd, 60);
|
||||||
|
} catch (Exception e) {
|
||||||
|
out = "(diagnostics sidecar failed: " + e + ")";
|
||||||
|
}
|
||||||
|
log.info("[netem-diag]\n{}", out);
|
||||||
|
log.info("===== END NETEM CONNECTIVITY DIAGNOSTICS =====");
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add delay + jitter to the target container's. OUTBOUND only: Pumba can only
|
* Add delay + jitter to the target container's. OUTBOUND only: Pumba can only
|
||||||
* delay with netem/tc, which is egress-only.
|
* delay with netem/tc, which is egress-only.
|
||||||
|
|
|
||||||
|
|
@ -783,6 +783,49 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dump everything needed to understand why the bridged (netem) PunchbagUser can't reach the SFU via
|
||||||
|
* the LiveKit URL. Logs both ends: (1) from the docker HOST, the SFU container's networks + the IPs
|
||||||
|
* it advertises for ICE (NODE_IP / the openvidu-pro container IP) and the host's own interfaces;
|
||||||
|
* (2) from INSIDE the netem container's netns, its IPs/routes/DNS and TCP/ICMP reachability of the
|
||||||
|
* LiveKit signaling host (see {@link NetworkConditioner#logConnectivityDiagnostics}); (3) best-effort
|
||||||
|
* browser console logs. Entirely best-effort -- never throws, so the real failure still surfaces.
|
||||||
|
*/
|
||||||
|
private void dumpNetemEstablishmentDiagnostics(OpenViduTestappUser user, String livekitUrl) {
|
||||||
|
try {
|
||||||
|
log.error("===== PunchbagUser establishment FAILED; dumping diagnostics. LiveKit URL = {} =====",
|
||||||
|
livekitUrl);
|
||||||
|
String hostPort = livekitUrl.replaceFirst("^wss?://", "").replaceFirst("/.*$", "");
|
||||||
|
String host = hostPort.contains(":") ? hostPort.substring(0, hostPort.indexOf(':')) : hostPort;
|
||||||
|
int port = hostPort.contains(":") ? Integer.parseInt(hostPort.substring(hostPort.indexOf(':') + 1)) : 7443;
|
||||||
|
|
||||||
|
// (1) SFU container view + host interfaces, from the docker host
|
||||||
|
log.error("[sfu-diag] docker inspect openvidu Networks ->\n{}",
|
||||||
|
commandLine.executeCommand("docker inspect -f '{{json .NetworkSettings.Networks}}' openvidu 2>&1", 30));
|
||||||
|
log.error("[sfu-diag] openvidu IP-related env (NODE_IP / LAN_*) ->\n{}", commandLine.executeCommand(
|
||||||
|
"docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' openvidu 2>&1 | grep -iE 'NODE_IP|LAN_|EXTERNAL|_IP' || true",
|
||||||
|
30));
|
||||||
|
log.error("[sfu-diag] docker host interfaces ->\n{}",
|
||||||
|
commandLine.executeCommand("ip -o addr 2>&1 || ifconfig -a 2>&1 || true", 30));
|
||||||
|
|
||||||
|
// (2) Netem container network-side probes (inside its own netns)
|
||||||
|
NetworkConditioner.logConnectivityDiagnostics(getNetemContainerName(user), host, port);
|
||||||
|
|
||||||
|
// (3) Browser console (best-effort; may be empty without goog:loggingPrefs)
|
||||||
|
try {
|
||||||
|
org.openqa.selenium.logging.LogEntries entries = user.getDriver().manage().logs()
|
||||||
|
.get(org.openqa.selenium.logging.LogType.BROWSER);
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
entries.forEach(en -> sb.append(en.getLevel()).append(' ').append(en.getMessage()).append('\n'));
|
||||||
|
log.error("[browser-console] PunchbagUser ->\n{}", sb.length() == 0 ? "(empty)" : sb.toString());
|
||||||
|
} catch (Exception ce) {
|
||||||
|
log.error("[browser-console] unavailable: {}", ce.getMessage());
|
||||||
|
}
|
||||||
|
} catch (Exception diagEx) {
|
||||||
|
log.error("Diagnostics dump itself failed (ignoring)", diagEx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String getConnectedSfuPortForPublisherPC(OpenViduTestappUser user, int numberOfUser)
|
private String getConnectedSfuPortForPublisherPC(OpenViduTestappUser user, int numberOfUser)
|
||||||
throws InterruptedException {
|
throws InterruptedException {
|
||||||
return extractConnectedSfuPort(readPcTransportsInfoJson(user, numberOfUser), "publisher");
|
return extractConnectedSfuPort(readPcTransportsInfoJson(user, numberOfUser), "publisher");
|
||||||
|
|
@ -883,7 +926,13 @@ public class OpenViduTestAppE2eTest extends AbstractOpenViduTestappE2eTest {
|
||||||
participantNameInput.sendKeys("PunchbagUser");
|
participantNameInput.sendKeys("PunchbagUser");
|
||||||
|
|
||||||
punchbagUser.getDriver().findElement(By.cssSelector(".connect-btn")).sendKeys(Keys.ENTER);
|
punchbagUser.getDriver().findElement(By.cssSelector(".connect-btn")).sendKeys(Keys.ENTER);
|
||||||
punchbagUser.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 1);
|
try {
|
||||||
|
punchbagUser.getEventManager().waitUntilEventReaches("connected", "RoomEvent", 1);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// The bridged (netem) PunchbagUser failed to establish (seen in CI). Dump why before failing.
|
||||||
|
dumpNetemEstablishmentDiagnostics(punchbagUser, secureLivekitUrlFromOpenViduLocalDeployment);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
// PunchbagUser always publishes now: audio+video as a publisher, audio only as
|
// PunchbagUser always publishes now: audio+video as a publisher, audio only as
|
||||||
// a subscriber.
|
// a subscriber.
|
||||||
punchbagUser.getEventManager().waitUntilEventReaches("localTrackPublished", "RoomEvent", isPublisher ? 2 : 1);
|
punchbagUser.getEventManager().waitUntilEventReaches("localTrackPublished", "RoomEvent", isPublisher ? 2 : 1);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue