From cf062c89509d5a4dd5554816b2d9281d83dda554 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Tue, 16 Feb 2021 11:23:39 +0100 Subject: [PATCH] openvidu-server: nodeCrashed event --- .../io/openvidu/server/cdr/CDREventName.java | 2 +- ...rCrashed.java => CDREventNodeCrashed.java} | 4 +-- .../openvidu/server/cdr/CallDetailRecord.java | 6 ++-- .../io/openvidu/server/core/EndReason.java | 2 +- .../server/core/SessionEventsHandler.java | 7 ++-- .../server/kurento/kms/KmsManager.java | 32 +++++++++++++------ .../OpenVidu.postman_collection.json | 2 +- .../src/main/resources/application.properties | 2 +- .../aplication-pro-webhook.properties | 3 +- 9 files changed, 39 insertions(+), 21 deletions(-) rename openvidu-server/src/main/java/io/openvidu/server/cdr/{CDREventMediaServerCrashed.java => CDREventNodeCrashed.java} (79%) diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventName.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventName.java index 6020e3e1..bef99490 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventName.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventName.java @@ -21,6 +21,6 @@ public enum CDREventName { sessionCreated, sessionDestroyed, participantJoined, participantLeft, webrtcConnectionCreated, webrtcConnectionDestroyed, recordingStarted, recordingStopped, recordingStatusChanged, filterEventDispatched, - signalSent, mediaNodeStatusChanged, autoscaling, mediaServerCrashed + signalSent, mediaNodeStatusChanged, autoscaling, nodeCrashed } diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventMediaServerCrashed.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventNodeCrashed.java similarity index 79% rename from openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventMediaServerCrashed.java rename to openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventNodeCrashed.java index fec4a96a..7061503b 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventMediaServerCrashed.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDREventNodeCrashed.java @@ -4,12 +4,12 @@ import com.google.gson.JsonObject; import io.openvidu.server.kurento.kms.Kms; -public class CDREventMediaServerCrashed extends CDREvent { +public class CDREventNodeCrashed extends CDREvent { private Kms kms; private String environmentId; - public CDREventMediaServerCrashed(CDREventName eventName, String sessionId, Long timeStamp, Kms kms, + public CDREventNodeCrashed(CDREventName eventName, String sessionId, Long timeStamp, Kms kms, String environmentId) { super(eventName, sessionId, timeStamp); this.kms = kms; diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java index 9ea6d0ef..259ffb6e 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java @@ -217,9 +217,9 @@ public class CallDetailRecord { }); } - public void recordMediaServerCrashed(Kms kms, String environmentId, long timeOfKurentoDisconnection) { - CDREvent e = new CDREventMediaServerCrashed(CDREventName.mediaServerCrashed, null, timeOfKurentoDisconnection, - kms, environmentId); + public void recordNodeCrashed(Kms kms, String environmentId, long timeOfKurentoDisconnection) { + CDREvent e = new CDREventNodeCrashed(CDREventName.nodeCrashed, null, timeOfKurentoDisconnection, kms, + environmentId); this.log(e); } diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/EndReason.java b/openvidu-server/src/main/java/io/openvidu/server/core/EndReason.java index 220a789a..6d002822 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/EndReason.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/EndReason.java @@ -20,7 +20,7 @@ package io.openvidu.server.core; public enum EndReason { unsubscribe, unpublish, disconnect, forceUnpublishByUser, forceUnpublishByServer, forceDisconnectByUser, - forceDisconnectByServer, lastParticipantLeft, networkDisconnect, mediaServerDisconnect, mediaServerCrashed, + forceDisconnectByServer, lastParticipantLeft, networkDisconnect, mediaServerDisconnect, nodeCrashed, openviduServerStopped, recordingStoppedByServer, automaticStop, sessionClosedByServer } diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java index e4c780bd..c085362b 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java @@ -607,8 +607,11 @@ public class SessionEventsHandler { public void onConnectionPropertyChanged(Participant participant, String property, Object newValue) { } - public void onMediaServerCrashed(Kms kms, long timeOfKurentoDisconnection) { - CDR.recordMediaServerCrashed(kms, null, timeOfKurentoDisconnection); + public void onMediaNodeCrashed(Kms kms, long timeOfKurentoDisconnection) { + CDR.recordNodeCrashed(kms, null, timeOfKurentoDisconnection); + } + + public void onMasterNodeCrashed() { } protected Set filterParticipantsByRole(OpenViduRole[] roles, Set participants) { diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/kms/KmsManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/kms/KmsManager.java index 11a05418..6918abf9 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/kms/KmsManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/kms/KmsManager.java @@ -198,29 +198,43 @@ public abstract class KmsManager { kms.getUri(), kms.getKurentoClient().toString()); } - final int loops = 6; + // 6 attempts, 2 times per second (3 seconds total) + final int maxReconnectTimeMillis = 3000; + final int intervalWaitMs = 500; + final int loops = maxReconnectTimeMillis / intervalWaitMs; final AtomicInteger iteration = new AtomicInteger(loops); - final long intervalWaitMs = 500L; + + final long initTime = System.currentTimeMillis(); final UpdatableTimerTask kurentoClientReconnectTimer = new UpdatableTimerTask(() -> { if (iteration.decrementAndGet() < 0) { - log.error("KurentoClient [{}] could not reconnect to KMS with uri {} in {} seconds", - kms.getKurentoClient().toString(), kms.getUri(), (intervalWaitMs * 6 / 1000)); + log.error( + "KurentoClient [{}] could not reconnect to KMS with uri {} in {} seconds. Media Node crashed", + kms.getKurentoClient().toString(), kms.getUri(), (intervalWaitMs * loops / 1000)); kms.getKurentoClientReconnectTimer().cancelTimer(); + + final long timeOfKurentoDisconnection = kms.getTimeOfKurentoClientDisconnection(); + sessionEventsHandler.onMediaNodeCrashed(kms, timeOfKurentoDisconnection); + log.warn("Closing {} sessions hosted by KMS with uri {}: {}", kms.getKurentoSessions().size(), kms.getUri(), kms.getKurentoSessions().stream().map(s -> s.getSessionId()) .collect(Collectors.joining(",", "[", "]"))); - final long timeOfKurentoDisconnection = kms.getTimeOfKurentoClientDisconnection(); - sessionEventsHandler.onMediaServerCrashed(kms, timeOfKurentoDisconnection); - kms.getKurentoSessions().forEach(kSession -> { - sessionManager.closeSession(kSession.getSessionId(), EndReason.mediaServerCrashed); + sessionManager.closeSession(kSession.getSessionId(), EndReason.nodeCrashed); }); } else { + // KurentoClient connection timeout may exceed the limit. + // This happens if not only kms process has crashed, but the instance itself is + // not reachable + if ((System.currentTimeMillis() - initTime) > maxReconnectTimeMillis) { + iteration.set(0); + return; + } + try { kms.getKurentoClient().getServerManager().getInfo(); } catch (Exception e) { @@ -256,7 +270,7 @@ public abstract class KmsManager { kms.setTimeOfKurentoClientDisconnection(0); } - }, () -> intervalWaitMs); // Try 2 times per seconds + }, () -> Long.valueOf(intervalWaitMs)); // Try 2 times per seconds kms.setKurentoClientReconnectTimer(kurentoClientReconnectTimer); kurentoClientReconnectTimer.updateTimer(); diff --git a/openvidu-server/src/main/resources/OpenVidu.postman_collection.json b/openvidu-server/src/main/resources/OpenVidu.postman_collection.json index 86399bcd..76b4f5dc 100644 --- a/openvidu-server/src/main/resources/OpenVidu.postman_collection.json +++ b/openvidu-server/src/main/resources/OpenVidu.postman_collection.json @@ -1097,7 +1097,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"OPENVIDU_SECRET\":\"MY_SECRET\",\n \"OPENVIDU_CDR\":true,\n \"OPENVIDU_RECORDING\":true,\n \"OPENVIDU_RECORDING_PUBLIC_ACCESS\":true,\n \"OPENVIDU_RECORDING_NOTIFICATION\":\"publisher_moderator\",\n \"OPENVIDU_RECORDING_PATH\":\"/opt/openvidu/recordings\",\n \"OPENVIDU_RECORDING_CUSTOM_LAYOUT\":\"/opt/openvidu/custom-layout\",\n \"OPENVIDU_RECORDING_AUTOSTOP_TIMEOUT\":120,\n \"OPENVIDU_WEBHOOK\":false,\n \"OPENVIDU_WEBHOOK_ENDPOINT\":\"http://localhost:7777/webhook/\",\n \"OPENVIDU_WEBHOOK_HEADERS\":[\n \"Authorization: Basic T1BFTlZJRFVBUFA6TVlfU0VDUkVU\"\n ],\n \"OPENVIDU_WEBHOOK_EVENTS\":[\n \"recordingStatusChanged\"\n ],\n \"OPENVIDU_STREAMS_VIDEO_MAX_RECV_BANDWIDTH\":1000,\n \"OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH\":300,\n \"OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH\":1000,\n \"OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH\":300,\n \"OPENVIDU_SESSIONS_GARBAGE_INTERVAL\":900,\n \"OPENVIDU_SESSIONS_GARBAGE_THRESHOLD\":3600,\n \"OPENVIDU_PRO_STATS_MONITORING_INTERVAL\":30,\n \"OPENVIDU_PRO_STATS_WEBRTC_INTERVAL\":20\n}" + "raw": "{\n \"OPENVIDU_SECRET\":\"MY_SECRET\",\n \"OPENVIDU_CDR\":true,\n \"OPENVIDU_RECORDING\":true,\n \"OPENVIDU_RECORDING_PUBLIC_ACCESS\":true,\n \"OPENVIDU_RECORDING_NOTIFICATION\":\"publisher_moderator\",\n \"OPENVIDU_RECORDING_PATH\":\"/opt/openvidu/recordings\",\n \"OPENVIDU_RECORDING_CUSTOM_LAYOUT\":\"/opt/openvidu/custom-layout\",\n \"OPENVIDU_RECORDING_AUTOSTOP_TIMEOUT\":120,\n \"OPENVIDU_WEBHOOK\":false,\n \"OPENVIDU_WEBHOOK_ENDPOINT\":\"http://localhost:7777/webhook/\",\n \"OPENVIDU_WEBHOOK_HEADERS\":[\n \"Authorization: Basic T1BFTlZJRFVBUFA6TVlfU0VDUkVU\"\n ],\n \"OPENVIDU_WEBHOOK_EVENTS\":[\n \"recordingStatusChanged\"\n ],\n \"OPENVIDU_STREAMS_VIDEO_MAX_RECV_BANDWIDTH\":1000,\n \"OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH\":300,\n \"OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH\":1000,\n \"OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH\":300,\n \"OPENVIDU_SESSIONS_GARBAGE_INTERVAL\":900,\n \"OPENVIDU_SESSIONS_GARBAGE_THRESHOLD\":3600,\n \"OPENVIDU_PRO_STATS_MONITORING_INTERVAL\":10,\n \"OPENVIDU_PRO_STATS_WEBRTC_INTERVAL\":20\n}" }, "url": { "raw": "https://localhost:4443/openvidu/api/restart", diff --git a/openvidu-server/src/main/resources/application.properties b/openvidu-server/src/main/resources/application.properties index 9cd079f5..7e105a0c 100644 --- a/openvidu-server/src/main/resources/application.properties +++ b/openvidu-server/src/main/resources/application.properties @@ -24,7 +24,7 @@ OPENVIDU_CDR_PATH=/opt/openvidu/cdr OPENVIDU_WEBHOOK=false OPENVIDU_WEBHOOK_ENDPOINT= OPENVIDU_WEBHOOK_HEADERS=[] -OPENVIDU_WEBHOOK_EVENTS=["sessionCreated","sessionDestroyed","participantJoined","participantLeft","webrtcConnectionCreated","webrtcConnectionDestroyed","recordingStatusChanged","filterEventDispatched","signalSent","mediaNodeStatusChanged","autoscaling","mediaServerCrashed"] +OPENVIDU_WEBHOOK_EVENTS=["sessionCreated","sessionDestroyed","participantJoined","participantLeft","webrtcConnectionCreated","webrtcConnectionDestroyed","recordingStatusChanged","filterEventDispatched","signalSent","mediaNodeStatusChanged","autoscaling","nodeCrashed"] OPENVIDU_RECORDING=false OPENVIDU_RECORDING_DEBUG=false diff --git a/openvidu-test-browsers/src/main/resources/aplication-pro-webhook.properties b/openvidu-test-browsers/src/main/resources/aplication-pro-webhook.properties index ff825235..7dbff853 100644 --- a/openvidu-test-browsers/src/main/resources/aplication-pro-webhook.properties +++ b/openvidu-test-browsers/src/main/resources/aplication-pro-webhook.properties @@ -1,2 +1,3 @@ server.port=7777 -server.ssl.enabled=false \ No newline at end of file +server.ssl.enabled=false +management.metrics.export.elastic.enabled=false \ No newline at end of file