From af57ce747ff9a313fac856eaaf93cee0d875906c Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Thu, 18 Feb 2021 16:02:03 +0100 Subject: [PATCH] openvidu-server: webrtcDebug event --- .../io/openvidu/client/OpenViduException.java | 2 +- .../io/openvidu/server/cdr/CDRLogger.java | 2 + .../io/openvidu/server/cdr/CDRLoggerFile.java | 4 + .../openvidu/server/cdr/CallDetailRecord.java | 6 ++ .../openvidu/server/cdr/WebrtcDebugEvent.java | 28 ++++-- .../kurento/core/KurentoParticipant.java | 20 +++-- .../kurento/core/KurentoSessionManager.java | 89 +++++++++++++++++-- .../kurento/endpoint/PublisherEndpoint.java | 4 + .../server/webhook/CDRLoggerWebhook.java | 5 ++ .../src/app/services/openvidu-rest.service.ts | 2 +- 10 files changed, 135 insertions(+), 27 deletions(-) diff --git a/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java b/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java index c39ece51..ed79e5ba 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java +++ b/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java @@ -36,7 +36,7 @@ public class OpenViduException extends JsonRpcErrorException { ROOM_CANNOT_BE_CREATED_ERROR_CODE(204), ROOM_CLOSED_ERROR_CODE(203), ROOM_NOT_FOUND_ERROR_CODE(202), ROOM_GENERIC_ERROR_CODE(201), - USER_NOT_STREAMING_ERROR_CODE(105), EXISTING_USER_IN_ROOM_ERROR_CODE(104), USER_CLOSED_ERROR_CODE(103), + USER_ALREADY_STREAMING_ERROR_CODE(106), USER_NOT_STREAMING_ERROR_CODE(105), EXISTING_USER_IN_ROOM_ERROR_CODE(104), USER_CLOSED_ERROR_CODE(103), USER_NOT_FOUND_ERROR_CODE(102), USER_GENERIC_ERROR_CODE(10), USER_UNAUTHORIZED_ERROR_CODE(401), ROLE_NOT_FOUND_ERROR_CODE(402), SESSIONID_CANNOT_BE_CREATED_ERROR_CODE(403), diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLogger.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLogger.java index b107c2b9..2112bf59 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLogger.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLogger.java @@ -26,6 +26,8 @@ public interface CDRLogger { public void log(KmsEvent event); + public void log(WebrtcDebugEvent event); + public void log(SessionSummary sessionSummary); } diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLoggerFile.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLoggerFile.java index 591bb17a..90b6200c 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLoggerFile.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CDRLoggerFile.java @@ -39,6 +39,10 @@ public class CDRLoggerFile implements CDRLogger { public void log(KmsEvent event) { } + @Override + public void log(WebrtcDebugEvent event) { + } + @Override public void log(SessionSummary sessionSummary) { } diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java index 259ffb6e..2d887da9 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/CallDetailRecord.java @@ -211,6 +211,12 @@ public class CallDetailRecord { }); } + public void log(WebrtcDebugEvent event) { + this.loggers.forEach(logger -> { + logger.log(event); + }); + } + public void log(SessionSummary sessionSummary) { this.loggers.forEach(logger -> { logger.log(sessionSummary); diff --git a/openvidu-server/src/main/java/io/openvidu/server/cdr/WebrtcDebugEvent.java b/openvidu-server/src/main/java/io/openvidu/server/cdr/WebrtcDebugEvent.java index 1e1e47a0..1ca66ff2 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/cdr/WebrtcDebugEvent.java +++ b/openvidu-server/src/main/java/io/openvidu/server/cdr/WebrtcDebugEvent.java @@ -6,16 +6,28 @@ import io.openvidu.server.core.Participant; public class WebrtcDebugEvent { + public enum WebrtcDebugEventIssuer { + client, server + } + + public enum WebrtcDebugEventOperation { + publish, subscribe, reconnectPublisher, reconnectSubscriber + } + + public enum WebrtcDebugEventType { + sdpOffer, sdpOfferMunged, sdpAnswer + } + private Participant participant; private String endpoint; - private String issuer; // [client, server] - private String operation; // [publish, subscribe, reconnectPublisher, reconnectSubscriber] - private String type; // [sdpOffer, mungedSdpOffer, sdpAnswer] + private WebrtcDebugEventIssuer issuer; + private WebrtcDebugEventOperation operation; + private WebrtcDebugEventType type; private String content; private Long timestamp; - public WebrtcDebugEvent(Participant participant, String endpoint, String issuer, String operation, String type, - String content) { + public WebrtcDebugEvent(Participant participant, String endpoint, WebrtcDebugEventIssuer issuer, + WebrtcDebugEventOperation operation, WebrtcDebugEventType type, String content) { this.participant = participant; this.endpoint = endpoint; this.issuer = issuer; @@ -31,9 +43,9 @@ public class WebrtcDebugEvent { json.addProperty("user", participant.getFinalUserId()); json.addProperty("connectionId", participant.getParticipantPublicId()); json.addProperty("endpoint", this.endpoint); - json.addProperty("issuer", this.issuer); - json.addProperty("operation", this.operation); - json.addProperty("type", this.type); + json.addProperty("issuer", this.issuer.name()); + json.addProperty("operation", this.operation.name()); + json.addProperty("type", this.type.name()); json.addProperty("content", this.content); json.addProperty("timestamp", this.timestamp); return json; diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java index f9cefaf6..db82c44e 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java @@ -99,12 +99,6 @@ public class KurentoParticipant extends Participant { } public void createPublishingEndpoint(MediaOptions mediaOptions, String streamId) { - String type = mediaOptions.hasVideo() ? mediaOptions.getTypeOfVideo() : "MICRO"; - if (streamId == null) { - streamId = IdentifierPrefixes.STREAM_ID + type.substring(0, Math.min(type.length(), 3)) + "_" - + RandomStringUtils.randomAlphabetic(1).toUpperCase() + RandomStringUtils.randomAlphanumeric(3) - + "_" + this.getParticipantPublicId(); - } publisher.setStreamId(streamId); publisher.setEndpointName(streamId); publisher.setMediaOptions(mediaOptions); @@ -261,8 +255,7 @@ public class KurentoParticipant extends Participant { "Unable to create subscriber endpoint"); } - String subscriberEndpointName = this.getParticipantPublicId() + "_" - + kSender.getPublisherStreamId(); + String subscriberEndpointName = calculateSubscriberEndpointName(kSender); subscriber.setEndpointName(subscriberEndpointName); subscriber.getEndpoint().setName(subscriberEndpointName); @@ -434,6 +427,17 @@ public class KurentoParticipant extends Participant { session.sendMediaError(this.getParticipantPrivateId(), desc); } + public String generateStreamId(MediaOptions mediaOptions) { + String type = mediaOptions.hasVideo() ? mediaOptions.getTypeOfVideo() : "MICRO"; + return IdentifierPrefixes.STREAM_ID + type.substring(0, Math.min(type.length(), 3)) + "_" + + RandomStringUtils.randomAlphabetic(1).toUpperCase() + RandomStringUtils.randomAlphanumeric(3) + "_" + + this.getParticipantPublicId(); + } + + public String calculateSubscriberEndpointName(Participant senderParticipant) { + return this.getParticipantPublicId() + "_" + senderParticipant.getPublisherStreamId(); + } + private void releasePublisherEndpoint(EndReason reason, Long kmsDisconnectionTime) { if (publisher != null && publisher.getEndpoint() != null) { final ReadWriteLock closingLock = publisher.closingLock; diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java index 1b4ea327..9ddf56b2 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java @@ -57,6 +57,11 @@ import io.openvidu.java.client.RecordingMode; import io.openvidu.java.client.RecordingProperties; import io.openvidu.java.client.SessionProperties; import io.openvidu.java.client.VideoCodec; +import io.openvidu.server.cdr.CallDetailRecord; +import io.openvidu.server.cdr.WebrtcDebugEvent; +import io.openvidu.server.cdr.WebrtcDebugEvent.WebrtcDebugEventIssuer; +import io.openvidu.server.cdr.WebrtcDebugEvent.WebrtcDebugEventOperation; +import io.openvidu.server.cdr.WebrtcDebugEvent.WebrtcDebugEventType; import io.openvidu.server.core.EndReason; import io.openvidu.server.core.FinalUser; import io.openvidu.server.core.IdentifierPrefixes; @@ -91,6 +96,9 @@ public class KurentoSessionManager extends SessionManager { @Autowired private SDPMunging sdpMunging; + @Autowired + private CallDetailRecord CDR; + @Override /* Protected by Session.closingLock.readLock */ public void joinRoom(Participant participant, String sessionId, Integer transactionId) { @@ -361,6 +369,14 @@ public class KurentoSessionManager extends SessionManager { Set participants = null; String sdpAnswer = null; + if (participant.isStreaming()) { + log.warn("PARTICIPANT {}: Request to publish media in session {} but user is already publishing", + participant.getParticipantPublicId(), participant.getSessionId()); + throw new OpenViduException(Code.USER_ALREADY_STREAMING_ERROR_CODE, + "User " + participant.getParticipantPublicId() + " is already publishing in session " + + participant.getSessionId()); + } + KurentoMediaOptions kurentoOptions = (KurentoMediaOptions) mediaOptions; KurentoParticipant kParticipant = (KurentoParticipant) participant; KurentoSession kSession = kParticipant.getSession(); @@ -368,10 +384,17 @@ public class KurentoSessionManager extends SessionManager { boolean isTranscodingAllowed = kSession.getSessionProperties().isTranscodingAllowed(); VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + final String streamId = kParticipant.generateStreamId(kurentoOptions); + + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.publish, WebrtcDebugEventType.sdpOffer, kurentoOptions.sdpOffer)); + // Modify sdp if forced codec is defined if (forcedVideoCodec != VideoCodec.NONE && !participant.isIpcam()) { kurentoOptions.sdpOffer = sdpMunging.forceCodec(participant, kurentoOptions.sdpOffer, kurentoOptions.isOffer, kSession, true, false, isTranscodingAllowed, forcedVideoCodec); + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.publish, WebrtcDebugEventType.sdpOfferMunged, kurentoOptions.sdpOffer)); } log.debug( "Request [PUBLISH_MEDIA] isOffer={} sdp={} " @@ -379,7 +402,7 @@ public class KurentoSessionManager extends SessionManager { kurentoOptions.isOffer, kurentoOptions.sdpOffer, kurentoOptions.doLoopback, kurentoOptions.rtspUri, participant.getParticipantPublicId()); - kParticipant.createPublishingEndpoint(mediaOptions, null); + kParticipant.createPublishingEndpoint(mediaOptions, streamId); /* * for (MediaElement elem : kurentoOptions.mediaElements) { @@ -413,6 +436,9 @@ public class KurentoSessionManager extends SessionManager { kSession.getSessionId(), mediaOptions, sdpAnswer, participants, transactionId, e); } + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.server, + WebrtcDebugEventOperation.publish, WebrtcDebugEventType.sdpAnswer, sdpAnswer)); + if (this.openviduConfig.isRecordingModuleEnabled() && MediaMode.ROUTED.equals(kSession.getSessionProperties().mediaMode()) && kSession.getActivePublishers() == 0) { @@ -518,6 +544,7 @@ public class KurentoSessionManager extends SessionManager { public void subscribe(Participant participant, String senderName, String sdpOffer, Integer transactionId) { String sdpAnswer = null; Session session = null; + try { log.debug("Request [SUBSCRIBE] remoteParticipant={} sdpOffer={} ({})", senderName, sdpOffer, participant.getParticipantPublicId()); @@ -525,14 +552,6 @@ public class KurentoSessionManager extends SessionManager { KurentoParticipant kParticipant = (KurentoParticipant) participant; session = ((KurentoParticipant) participant).getSession(); Participant senderParticipant = session.getParticipantByPublicId(senderName); - boolean isTranscodingAllowed = session.getSessionProperties().isTranscodingAllowed(); - VideoCodec forcedVideoCodec = session.getSessionProperties().forcedVideoCodec(); - - // Modify sdp if forced codec is defined - if (forcedVideoCodec != VideoCodec.NONE && !participant.isIpcam()) { - sdpOffer = sdpMunging.forceCodec(participant, sdpOffer, true, session, false, false, - isTranscodingAllowed, forcedVideoCodec); - } if (senderParticipant == null) { log.warn( @@ -551,12 +570,33 @@ public class KurentoSessionManager extends SessionManager { "User '" + senderName + " not streaming media in session '" + session.getSessionId() + "'"); } + String subscriberEndpointName = kParticipant.calculateSubscriberEndpointName(senderParticipant); + + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.subscribe, WebrtcDebugEventType.sdpOffer, sdpOffer)); + + boolean isTranscodingAllowed = session.getSessionProperties().isTranscodingAllowed(); + VideoCodec forcedVideoCodec = session.getSessionProperties().forcedVideoCodec(); + + // Modify sdp if forced codec is defined + if (forcedVideoCodec != VideoCodec.NONE && !participant.isIpcam()) { + sdpOffer = sdpMunging.forceCodec(participant, sdpOffer, true, session, false, false, + isTranscodingAllowed, forcedVideoCodec); + + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.subscribe, WebrtcDebugEventType.sdpOfferMunged, sdpOffer)); + } + sdpAnswer = kParticipant.receiveMediaFrom(senderParticipant, sdpOffer, false); if (sdpAnswer == null) { throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, "Unable to generate SDP answer when subscribing '" + participant.getParticipantPublicId() + "' to '" + senderName + "'"); } + + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.server, + WebrtcDebugEventOperation.subscribe, WebrtcDebugEventType.sdpAnswer, sdpAnswer)); + } catch (OpenViduException e) { log.error("PARTICIPANT {}: Error subscribing to {}", participant.getParticipantPublicId(), senderName, e); sessionEventsHandler.onSubscribe(participant, session, null, transactionId, e); @@ -1077,14 +1117,25 @@ public class KurentoSessionManager extends SessionManager { boolean isTranscodingAllowed = kSession.getSessionProperties().isTranscodingAllowed(); VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + boolean sdpOfferHasBeenMunged = false; + String originalSdpOffer = sdpOffer; + // Modify sdp if forced codec is defined if (forcedVideoCodec != VideoCodec.NONE && !participant.isIpcam()) { + sdpOfferHasBeenMunged = true; sdpOffer = sdpMunging.forceCodec(participant, sdpOffer, true, kSession, isPublisher, true, isTranscodingAllowed, forcedVideoCodec); } if (isPublisher) { + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.reconnectPublisher, WebrtcDebugEventType.sdpOffer, originalSdpOffer)); + if (sdpOfferHasBeenMunged) { + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.reconnectPublisher, WebrtcDebugEventType.sdpOfferMunged, sdpOffer)); + } + // Reconnect publisher final KurentoMediaOptions kurentoOptions = (KurentoMediaOptions) kParticipant.getPublisher() .getMediaOptions(); @@ -1104,6 +1155,10 @@ public class KurentoSessionManager extends SessionManager { String sdpAnswer = kParticipant.publishToRoom(sdpType, sdpOffer, kurentoOptions.doLoopback, true); log.debug("SDP Answer for publishing reconnection PARTICIPANT {}: {}", participant.getParticipantPublicId(), sdpAnswer); + + CDR.log(new WebrtcDebugEvent(participant, streamId, WebrtcDebugEventIssuer.server, + WebrtcDebugEventOperation.reconnectPublisher, WebrtcDebugEventType.sdpAnswer, sdpAnswer)); + sessionEventsHandler.onPublishMedia(participant, participant.getPublisherStreamId(), kParticipant.getPublisher().createdAt(), kSession.getSessionId(), kurentoOptions, sdpAnswer, new HashSet(), transactionId, null); @@ -1113,7 +1168,19 @@ public class KurentoSessionManager extends SessionManager { // Reconnect subscriber String senderPrivateId = kSession.getParticipantPrivateIdFromStreamId(streamId); if (senderPrivateId != null) { + KurentoParticipant sender = (KurentoParticipant) kSession.getParticipantByPrivateId(senderPrivateId); + String subscriberEndpointName = kParticipant.calculateSubscriberEndpointName(sender); + + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.reconnectSubscriber, WebrtcDebugEventType.sdpOffer, + originalSdpOffer)); + if (sdpOfferHasBeenMunged) { + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.client, + WebrtcDebugEventOperation.reconnectSubscriber, WebrtcDebugEventType.sdpOfferMunged, + sdpOffer)); + } + kParticipant.cancelReceivingMedia(sender, null, true); String sdpAnswer = kParticipant.receiveMediaFrom(sender, sdpOffer, true); if (sdpAnswer == null) { @@ -1123,6 +1190,10 @@ public class KurentoSessionManager extends SessionManager { log.debug("SDP Answer for subscribing reconnection PARTICIPANT {}: {}", participant.getParticipantPublicId(), sdpAnswer); + + CDR.log(new WebrtcDebugEvent(participant, subscriberEndpointName, WebrtcDebugEventIssuer.server, + WebrtcDebugEventOperation.reconnectSubscriber, WebrtcDebugEventType.sdpAnswer, sdpAnswer)); + sessionEventsHandler.onSubscribe(participant, kSession, sdpAnswer, transactionId, null); } else { throw new OpenViduException(Code.USER_NOT_STREAMING_ERROR_CODE, diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java index e35de370..40fbeffb 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java @@ -201,7 +201,11 @@ public class PublisherEndpoint extends MediaEndpoint { String sdpResponse = null; switch (sdpType) { case ANSWER: + + /** THIS IS CURRENTLY NEVER CALLED **/ sdpResponse = processAnswer(sdpString); + /** THIS IS CURRENTLY NEVER CALLED **/ + break; case OFFER: sdpResponse = processOffer(sdpString); diff --git a/openvidu-server/src/main/java/io/openvidu/server/webhook/CDRLoggerWebhook.java b/openvidu-server/src/main/java/io/openvidu/server/webhook/CDRLoggerWebhook.java index acca20ae..aeb9fdeb 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/webhook/CDRLoggerWebhook.java +++ b/openvidu-server/src/main/java/io/openvidu/server/webhook/CDRLoggerWebhook.java @@ -24,6 +24,7 @@ import org.apache.http.Header; import io.openvidu.server.cdr.CDREvent; import io.openvidu.server.cdr.CDREventName; import io.openvidu.server.cdr.CDRLogger; +import io.openvidu.server.cdr.WebrtcDebugEvent; import io.openvidu.server.kurento.endpoint.KmsEvent; import io.openvidu.server.summary.SessionSummary; @@ -44,6 +45,10 @@ public class CDRLoggerWebhook implements CDRLogger { public void log(KmsEvent event) { } + @Override + public void log(WebrtcDebugEvent event) { + } + @Override public void log(SessionSummary sessionSummary) { } diff --git a/openvidu-testapp/src/app/services/openvidu-rest.service.ts b/openvidu-testapp/src/app/services/openvidu-rest.service.ts index 12668741..a497be80 100644 --- a/openvidu-testapp/src/app/services/openvidu-rest.service.ts +++ b/openvidu-testapp/src/app/services/openvidu-rest.service.ts @@ -49,7 +49,7 @@ export class OpenviduRestService { getAvailableParams(): Map { const params = new Map(); - this.sessionIdSession.forEach((sessionApi, sessionId, map) => { + this.sessionIdSession.forEach((sessionApi, sessionId) => { params.set(sessionId, Array.from(this.sessionIdTokenOpenViduRole.get(sessionId).keys())); }); return params;