From 0d3d54351e3a6e01123cedee3da7797d71d08371 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Tue, 1 Sep 2020 12:53:32 +0200 Subject: [PATCH 1/7] openvidu-browser: mediasoup refactoring --- openvidu-browser/src/OpenVidu/Stream.ts | 107 +++++++---- .../OpenViduInternal/WebRtcPeer/WebRtcPeer.ts | 170 ++++++++++++------ 2 files changed, 190 insertions(+), 87 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index e4f3cfe5..b3386c5f 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -822,7 +822,7 @@ export class Stream extends EventDispatcher { simulcast: false }; - const successCallback = (sdpOfferParam) => { + const successOfferCallback = (sdpOfferParam) => { logger.debug('Sending SDP offer to publish as ' + this.streamId, sdpOfferParam); @@ -859,7 +859,7 @@ export class Stream extends EventDispatcher { reject('Error on publishVideo: ' + JSON.stringify(error)); } } else { - this.webRtcPeer.processAnswer(response.sdpAnswer, false) + this.webRtcPeer.processRemoteAnswer(response.sdpAnswer) .then(() => { this.streamId = response.id; this.creationTime = response.createdAt; @@ -894,10 +894,15 @@ export class Stream extends EventDispatcher { this.webRtcPeer = new WebRtcPeerSendonly(options); } this.webRtcPeer.addIceConnectionStateChangeListener('publisher of ' + this.connection.connectionId); - this.webRtcPeer.generateOffer().then(sdpOffer => { - successCallback(sdpOffer); + this.webRtcPeer.createOffer().then(sdpOffer => { + this.webRtcPeer.processLocalOffer(sdpOffer) + .then(() => { + successOfferCallback(sdpOffer.sdp); + }).catch(error => { + reject(new Error('(publish) SDP process local offer error: ' + JSON.stringify(error))); + }); }).catch(error => { - reject(new Error('(publish) SDP offer error: ' + JSON.stringify(error))); + reject(new Error('(publish) SDP create offer error: ' + JSON.stringify(error))); }); }); } @@ -906,6 +911,30 @@ export class Stream extends EventDispatcher { * @hidden */ initWebRtcPeerReceive(reconnect: boolean): Promise { + return new Promise((resolve, reject) => { + this.session.openvidu.sendRequest('prepareReceiveVideFrom', { sender: this.streamId }, (error, response) => { + if (error) { + reject(new Error('Error on prepareReceiveVideFrom: ' + JSON.stringify(error))); + } else { + this.completeWebRtcPeerReceive(response.sdpOffer, reconnect) + .then(() => { + logger.info("'Subscriber' (" + this.streamId + ") successfully " + (reconnect ? "reconnected" : "subscribed")); + this.remotePeerSuccessfullyEstablished(); + this.initWebRtcStats(); + resolve(); + }) + .catch(error => { + reject(error); + }); + } + }); + }); + } + + /** + * @hidden + */ + completeWebRtcPeerReceive(sdpOffer: string, reconnect: boolean): Promise { return new Promise((resolve, reject) => { const offerConstraints = { @@ -921,50 +950,60 @@ export class Stream extends EventDispatcher { simulcast: false }; - const successCallback = (sdpOfferParam) => { - logger.debug('Sending SDP offer to subscribe to ' - + this.streamId, sdpOfferParam); + const successAnswerCallback = (sdpAnswer) => { + logger.debug('Sending SDP answer to subscribe to ' + + this.streamId, sdpAnswer); const method = reconnect ? 'reconnectStream' : 'receiveVideoFrom'; - const params = { sdpOffer: sdpOfferParam }; + const params = { sdpAnswer }; params[reconnect ? 'stream' : 'sender'] = this.streamId; this.session.openvidu.sendRequest(method, params, (error, response) => { if (error) { - reject(new Error('Error on recvVideoFrom: ' + JSON.stringify(error))); + reject(new Error('Error on receiveVideFrom: ' + JSON.stringify(error))); } else { - // Ios Ionic. Limitation: some bug in iosrtc cordova plugin makes it necessary - // to add a timeout before calling PeerConnection#setRemoteDescription during - // some time (400 ms) from the moment first subscriber stream is received - if (this.session.isFirstIonicIosSubscriber) { - this.session.isFirstIonicIosSubscriber = false; - setTimeout(() => { - // After 400 ms Ionic iOS subscribers won't need to run - // PeerConnection#setRemoteDescription after 250 ms timeout anymore - this.session.countDownForIonicIosSubscribersActive = false; - }, 400); - } - const needsTimeoutOnProcessAnswer = this.session.countDownForIonicIosSubscribersActive; - this.webRtcPeer.processAnswer(response.sdpAnswer, needsTimeoutOnProcessAnswer).then(() => { - logger.info("'Subscriber' (" + this.streamId + ") successfully " + (reconnect ? "reconnected" : "subscribed")); - this.remotePeerSuccessfullyEstablished(); - this.initWebRtcStats(); - resolve(); - }).catch(error => { - reject(error); - }); + resolve(); + // // Ios Ionic. Limitation: some bug in iosrtc cordova plugin makes it necessary + // // to add a timeout before calling PeerConnection#setRemoteDescription during + // // some time (400 ms) from the moment first subscriber stream is received + // if (this.session.isFirstIonicIosSubscriber) { + // this.session.isFirstIonicIosSubscriber = false; + // setTimeout(() => { + // // After 400 ms Ionic iOS subscribers won't need to run + // // PeerConnection#setRemoteDescription after 250 ms timeout anymore + // this.session.countDownForIonicIosSubscribersActive = false; + // }, 400); + // } + // const needsTimeoutOnProcessAnswer = this.session.countDownForIonicIosSubscribersActive; + // this.webRtcPeer.processAnswer(response.sdpAnswer, needsTimeoutOnProcessAnswer).then(() => { + // logger.info("'Subscriber' (" + this.streamId + ") successfully " + (reconnect ? "reconnected" : "subscribed")); + // this.remotePeerSuccessfullyEstablished(); + // this.initWebRtcStats(); + // resolve(); + // }).catch(error => { + // reject(error); + // }); } }); }; this.webRtcPeer = new WebRtcPeerRecvonly(options); this.webRtcPeer.addIceConnectionStateChangeListener(this.streamId); - this.webRtcPeer.generateOffer() - .then(sdpOffer => { - successCallback(sdpOffer); + this.webRtcPeer.processRemoteOffer(sdpOffer) + .then(() => { + this.webRtcPeer.createAnswer().then(sdpAnswer => { + this.webRtcPeer.processLocalAnswer(sdpAnswer) + .then(() => { + successAnswerCallback(sdpAnswer.sdp); + }).catch(error => { + reject(new Error('(subscribe) SDP process local answer error: ' + JSON.stringify(error))); + }); + }).catch(error => { + reject(new Error('(subscribe) SDP create answer error: ' + JSON.stringify(error))); + }); }) .catch(error => { - reject(new Error('(subscribe) SDP offer error: ' + JSON.stringify(error))); + reject(new Error('(subscribe) SDP process remote offer error: ' + JSON.stringify(error))); }); }); } diff --git a/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts b/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts index 42e161ae..c4141973 100644 --- a/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts +++ b/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts @@ -117,10 +117,10 @@ export class WebRtcPeer { } /** - * Function that creates an offer, sets it as local description and returns the offer param - * to send to OpenVidu Server (will be the remote description of other peer) + * Creates an SDP offer from the local RTCPeerConnection to send to the other peer + * Only if the negotiation was initiated by the this peer */ - generateOffer(): Promise { + createOffer(): Promise { return new Promise((resolve, reject) => { let offerAudio, offerVideo = true; @@ -140,52 +140,32 @@ export class WebRtcPeer { logger.debug('RTCPeerConnection constraints: ' + JSON.stringify(constraints)); if (platform.name === 'Safari' && platform.ua!!.indexOf('Safari') !== -1) { + // Safari (excluding Ionic), at least on iOS just seems to support unified plan, whereas in other browsers is not yet ready and considered experimental if (offerAudio) { this.pc.addTransceiver('audio', { direction: this.configuration.mode, }); } - if (offerVideo) { this.pc.addTransceiver('video', { direction: this.configuration.mode, }); } - - this.pc - .createOffer() + this.pc.createOffer() .then(offer => { logger.debug('Created SDP offer'); - return this.pc.setLocalDescription(offer); - }) - .then(() => { - const localDescription = this.pc.localDescription; - - if (!!localDescription) { - logger.debug('Local description set', localDescription.sdp); - resolve(localDescription.sdp); - } else { - reject('Local description is not defined'); - } + resolve(offer); }) .catch(error => reject(error)); } else { // Rest of platforms - this.pc.createOffer(constraints).then(offer => { - logger.debug('Created SDP offer'); - return this.pc.setLocalDescription(offer); - }) - .then(() => { - const localDescription = this.pc.localDescription; - if (!!localDescription) { - logger.debug('Local description set', localDescription.sdp); - resolve(localDescription.sdp); - } else { - reject('Local description is not defined'); - } + this.pc.createOffer(constraints) + .then(offer => { + logger.debug('Created SDP offer'); + resolve(offer); }) .catch(error => reject(error)); } @@ -193,10 +173,95 @@ export class WebRtcPeer { } /** - * Function invoked when a SDP answer is received. Final step in SDP negotiation, the peer - * just needs to set the answer as its remote description + * Creates an SDP answer from the local RTCPeerConnection to send to the other peer + * Only if the negotiation was initiated by the other peer */ - processAnswer(sdpAnswer: string, needsTimeoutOnProcessAnswer: boolean): Promise { + createAnswer(): Promise { + return new Promise((resolve, reject) => { + let offerAudio, offerVideo = true; + if (!!this.configuration.mediaConstraints) { + offerAudio = (typeof this.configuration.mediaConstraints.audio === 'boolean') ? + this.configuration.mediaConstraints.audio : true; + offerVideo = (typeof this.configuration.mediaConstraints.video === 'boolean') ? + this.configuration.mediaConstraints.video : true; + } + const constraints: RTCOfferOptions = { + offerToReceiveAudio: offerAudio, + offerToReceiveVideo: offerVideo + }; + this.pc.createAnswer(constraints).then(sdpAnswer => { + resolve(sdpAnswer); + }).catch(error => { + reject(error); + }); + }); + } + + /** + * This peer initiated negotiation. Step 1/4 of SDP offer-answer protocol + */ + processLocalOffer(offer: RTCSessionDescriptionInit): Promise { + return new Promise((resolve, reject) => { + this.pc.setLocalDescription(offer) + .then(() => { + const localDescription = this.pc.localDescription; + if (!!localDescription) { + logger.debug('Local description set', localDescription.sdp); + resolve(); + } else { + reject('Local description is not defined'); + } + }) + .catch(error => { + reject(error); + }); + }); + } + + /** + * Other peer initiated negotiation. Step 2/4 of SDP offer-answer protocol + */ + processRemoteOffer(sdpOffer: string): Promise { + return new Promise((resolve, reject) => { + const offer: RTCSessionDescriptionInit = { + type: 'offer', + sdp: sdpOffer + }; + logger.debug('SDP offer received, setting remote description', offer); + + if (this.pc.signalingState === 'closed') { + reject('RTCPeerConnection is closed when trying to set remote description'); + } + // TODO: check if Ionic iOS still needs timeout on setting first remote description when subscribing + this.setRemoteDescription(offer, false) + .then(() => { + resolve(); + }) + .catch(error => { + reject(error); + }); + }); + } + + /** + * Other peer initiated negotiation. Step 3/4 of SDP offer-answer protocol + */ + processLocalAnswer(answer: RTCSessionDescriptionInit): Promise { + return new Promise((resolve, reject) => { + logger.debug('SDP answer created, setting local description'); + if (this.pc.signalingState === 'closed') { + reject('RTCPeerConnection is closed when trying to set local description'); + } + this.pc.setLocalDescription(answer) + .then(() => resolve()) + .catch(error => reject(error)); + }); + } + + /** + * This peer initiated negotiation. Step 4/4 of SDP offer-answer protocol + */ + processRemoteAnswer(sdpAnswer: string): Promise { return new Promise((resolve, reject) => { const answer: RTCSessionDescriptionInit = { type: 'answer', @@ -205,34 +270,33 @@ export class WebRtcPeer { logger.debug('SDP answer received, setting remote description'); if (this.pc.signalingState === 'closed') { - reject('RTCPeerConnection is closed'); + reject('RTCPeerConnection is closed when trying to set remote description'); } - - this.setRemoteDescription(answer, needsTimeoutOnProcessAnswer, resolve, reject); - + this.setRemoteDescription(answer, false) + .then(() => resolve()) + .catch(error => reject(error)); }); } /** * @hidden */ - setRemoteDescription(answer: RTCSessionDescriptionInit, needsTimeoutOnProcessAnswer: boolean, resolve: (value?: string | PromiseLike | undefined) => void, reject: (reason?: any) => void) { - if (platform['isIonicIos']) { - // Ionic iOS platform - if (needsTimeoutOnProcessAnswer) { - // 400 ms have not elapsed yet since first remote stream triggered Stream#initWebRtcPeerReceive - setTimeout(() => { - logger.info('setRemoteDescription run after timeout for Ionic iOS device'); - this.pc.setRemoteDescription(new RTCSessionDescription(answer)).then(() => resolve()).catch(error => reject(error)); - }, 250); - } else { - // 400 ms have elapsed - this.pc.setRemoteDescription(new RTCSessionDescription(answer)).then(() => resolve()).catch(error => reject(error)); - } - } else { + async setRemoteDescription(sdp: RTCSessionDescriptionInit, needsTimeoutOnProcessAnswer: boolean): Promise { + // if (platform['isIonicIos']) { + // // Ionic iOS platform + // if (needsTimeoutOnProcessAnswer) { + // // 400 ms have not elapsed yet since first remote stream triggered Stream#initWebRtcPeerReceive + // await new Promise(resolve => setTimeout(resolve, 250)); // Sleep for 250ms + // logger.info('setRemoteDescription run after timeout for Ionic iOS device'); + // return this.pc.setRemoteDescription(sdp); + // } else { + // // 400 ms have elapsed + // return this.pc.setRemoteDescription(sdp); + // } + // } else { // Rest of platforms - this.pc.setRemoteDescription(answer).then(() => resolve()).catch(error => reject(error)); - } + return this.pc.setRemoteDescription(sdp); + // } } /** From 3086b4aaf27d79e4339f2935e88ee5bf9f4cb865 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Tue, 1 Sep 2020 15:17:17 +0200 Subject: [PATCH 2/7] openvidu-server: mediasoup refactoring --- openvidu-browser/src/OpenVidu/Stream.ts | 10 +- .../client/internal/ProtocolElements.java | 5 +- .../server/core/SessionEventsHandler.java | 14 +- .../openvidu/server/core/SessionManager.java | 4 +- .../kurento/core/KurentoParticipant.java | 75 +++++++--- .../kurento/core/KurentoSessionManager.java | 86 +++++++---- .../kurento/endpoint/MediaEndpoint.java | 18 +++ .../kurento/endpoint/PublisherEndpoint.java | 134 +++++++++--------- .../kurento/endpoint/SubscriberEndpoint.java | 22 +-- .../server/recording/CompositeWrapper.java | 2 +- .../service/SingleStreamRecordingService.java | 8 +- .../io/openvidu/server/rpc/RpcHandler.java | 56 +++++--- 12 files changed, 277 insertions(+), 157 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index b3386c5f..9178f3af 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -830,7 +830,8 @@ export class Stream extends EventDispatcher { let params; if (reconnect) { params = { - stream: this.streamId + stream: this.streamId, + sdpString: sdpOfferParam } } else { let typeOfVideo = ''; @@ -846,10 +847,10 @@ export class Stream extends EventDispatcher { typeOfVideo, frameRate: !!this.frameRate ? this.frameRate : -1, videoDimensions: JSON.stringify(this.videoDimensions), - filter: this.outboundStreamOpts.publisherProperties.filter + filter: this.outboundStreamOpts.publisherProperties.filter, + sdpOffer: sdpOfferParam } } - params['sdpOffer'] = sdpOfferParam; this.session.openvidu.sendRequest(method, params, (error, response) => { if (error) { @@ -955,8 +956,9 @@ export class Stream extends EventDispatcher { + this.streamId, sdpAnswer); const method = reconnect ? 'reconnectStream' : 'receiveVideoFrom'; - const params = { sdpAnswer }; + const params = {}; params[reconnect ? 'stream' : 'sender'] = this.streamId; + params[reconnect ? 'sdpString' : 'sdpAnswer'] = sdpAnswer; this.session.openvidu.sendRequest(method, params, (error, response) => { if (error) { diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index 8f033494..5e4d62c9 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -70,6 +70,9 @@ public class ProtocolElements { public static final String UNPUBLISHVIDEO_METHOD = "unpublishVideo"; + public static final String PREPARERECEIVEVIDEO_METHOD = "prepareReceiveVideFrom"; + public static final String PREPARERECEIVEVIDEO_SDPOFFER_PARAM = "sdpOffer"; + public static final String RECEIVEVIDEO_METHOD = "receiveVideoFrom"; public static final String RECEIVEVIDEO_SDPOFFER_PARAM = "sdpOffer"; public static final String RECEIVEVIDEO_SENDER_PARAM = "sender"; @@ -120,7 +123,7 @@ public class ProtocolElements { public static final String RECONNECTSTREAM_METHOD = "reconnectStream"; public static final String RECONNECTSTREAM_STREAM_PARAM = "stream"; - public static final String RECONNECTSTREAM_SDPOFFER_PARAM = "sdpOffer"; + public static final String RECONNECTSTREAM_SDPSTRING_PARAM = "sdpString"; // ---------------------------- SERVER RESPONSES & EVENTS ----------------- diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java index e217cff4..34609036 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java @@ -279,14 +279,24 @@ public class SessionEventsHandler { } } - public void onSubscribe(Participant participant, Session session, String sdpAnswer, Integer transactionId, + public void onPrepareSubscription(Participant participant, Session session, String sdpOffer, Integer transactionId, + OpenViduException error) { + if (error != null) { + rpcNotificationService.sendErrorResponse(participant.getParticipantPrivateId(), transactionId, null, error); + return; + } + JsonObject result = new JsonObject(); + result.addProperty(ProtocolElements.PREPARERECEIVEVIDEO_SDPOFFER_PARAM, sdpOffer); + rpcNotificationService.sendResponse(participant.getParticipantPrivateId(), transactionId, result); + } + + public void onSubscribe(Participant participant, Session session, Integer transactionId, OpenViduException error) { if (error != null) { rpcNotificationService.sendErrorResponse(participant.getParticipantPrivateId(), transactionId, null, error); return; } JsonObject result = new JsonObject(); - result.addProperty(ProtocolElements.RECEIVEVIDEO_SDPANSWER_PARAM, sdpAnswer); rpcNotificationService.sendResponse(participant.getParticipantPrivateId(), transactionId, result); if (ProtocolElements.RECORDER_PARTICIPANT_PUBLICID.equals(participant.getParticipantPublicId())) { diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java index 91dd8fd6..af55da6e 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java @@ -106,7 +106,9 @@ public abstract class SessionManager { public abstract void unpublishVideo(Participant participant, Participant moderator, Integer transactionId, EndReason reason); - public abstract void subscribe(Participant participant, String senderName, String sdpOffer, Integer transactionId); + public abstract void prepareSubscription(Participant participant, String senderPublicId, Integer id); + + public abstract void subscribe(Participant participant, String senderName, String sdpAnwser, Integer transactionId); public abstract void unsubscribe(Participant participant, String senderName, Integer transactionId); diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java index 05c795b3..45017756 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoParticipant.java @@ -54,7 +54,6 @@ import io.openvidu.server.core.MediaOptions; import io.openvidu.server.core.Participant; import io.openvidu.server.kurento.endpoint.MediaEndpoint; import io.openvidu.server.kurento.endpoint.PublisherEndpoint; -import io.openvidu.server.kurento.endpoint.SdpType; import io.openvidu.server.kurento.endpoint.SubscriberEndpoint; import io.openvidu.server.recording.service.RecordingManager; @@ -169,15 +168,15 @@ public class KurentoParticipant extends Participant { return session; } - public String publishToRoom(SdpType sdpType, String sdpString, boolean doLoopback, boolean silent) { - log.info("PARTICIPANT {}: Request to publish video in room {} (sdp type {})", this.getParticipantPublicId(), - this.session.getSessionId(), sdpType); - log.trace("PARTICIPANT {}: Publishing Sdp ({}) is {}", this.getParticipantPublicId(), sdpType, sdpString); + public String publishToRoom(String sdpOffer, boolean doLoopback, boolean silent) { + log.info("PARTICIPANT {}: Request to publish video in room {})", this.getParticipantPublicId(), + this.session.getSessionId()); + log.trace("PARTICIPANT {}: Publishing Sdp Offer is {}", this.getParticipantPublicId(), sdpOffer); - String sdpResponse = this.getPublisher().publish(sdpType, sdpString, doLoopback); + String sdpAnswer = this.getPublisher().publish(sdpOffer, doLoopback); this.streaming = true; - log.trace("PARTICIPANT {}: Publishing Sdp ({}) is {}", this.getParticipantPublicId(), sdpType, sdpResponse); + log.trace("PARTICIPANT {}: Publishing Sdp Answer is {}", this.getParticipantPublicId(), sdpAnswer); log.info("PARTICIPANT {}: Is now publishing video in room {}", this.getParticipantPublicId(), this.session.getSessionId()); @@ -191,7 +190,7 @@ public class KurentoParticipant extends Participant { publisher.getMediaOptions(), publisher.createdAt()); } - return sdpResponse; + return sdpAnswer; } public void unpublishMedia(EndReason reason, long kmsDisconnectionTime) { @@ -204,12 +203,11 @@ public class KurentoParticipant extends Participant { this.getParticipantPublicId()); } - public String receiveMediaFrom(Participant sender, String sdpOffer, boolean silent) { + public String prepareReceiveMediaFrom(Participant sender) { final String senderName = sender.getParticipantPublicId(); - log.info("PARTICIPANT {}: Request to receive media from {} in room {}", this.getParticipantPublicId(), + log.info("PARTICIPANT {}: Request to prepare receive media from {} in room {}", this.getParticipantPublicId(), senderName, this.session.getSessionId()); - log.trace("PARTICIPANT {}: SdpOffer for {} is {}", this.getParticipantPublicId(), senderName, sdpOffer); if (senderName.equals(this.getParticipantPublicId())) { log.warn("PARTICIPANT {}: trying to configure loopback by subscribing", this.getParticipantPublicId()); @@ -269,8 +267,56 @@ public class KurentoParticipant extends Participant { log.debug("PARTICIPANT {}: Created subscriber endpoint for user {}", this.getParticipantPublicId(), senderName); try { - String sdpAnswer = subscriber.subscribe(sdpOffer, kSender.getPublisher()); - log.trace("PARTICIPANT {}: Subscribing SdpAnswer is {}", this.getParticipantPublicId(), sdpAnswer); + String sdpOffer = subscriber.prepareSubscription(kSender.getPublisher()); + log.trace("PARTICIPANT {}: Subscribing SdpOffer is {}", this.getParticipantPublicId(), sdpOffer); + log.info("PARTICIPANT {}: offer prepared to receive media from {} in room {}", + this.getParticipantPublicId(), senderName, this.session.getSessionId()); + return sdpOffer; + } catch (KurentoServerException e) { + log.error("Exception preparing subscriber endpoint for user {}: {}", this.getParticipantPublicId(), + e.getMessage()); + this.subscribers.remove(senderName); + releaseSubscriberEndpoint(senderName, (KurentoParticipant) sender, subscriber, null, false); + return null; + } + } finally { + kSender.getPublisher().closingLock.readLock().unlock(); + } + } else { + log.error( + "PublisherEndpoint of participant {} of session {} is closed. Participant {} couldn't subscribe to it ", + senderName, sender.getSessionId(), this.participantPublicId); + throw new OpenViduException(Code.MEDIA_ENDPOINT_ERROR_CODE, + "Unable to create subscriber endpoint. Publisher endpoint of participant " + senderName + + "is closed"); + } + } + + public void receiveMediaFrom(Participant sender, String sdpAnswer, boolean silent) { + final String senderName = sender.getParticipantPublicId(); + + log.info("PARTICIPANT {}: Request to receive media from {} in room {}", this.getParticipantPublicId(), + senderName, this.session.getSessionId()); + log.trace("PARTICIPANT {}: SdpAnswer for {} is {}", this.getParticipantPublicId(), senderName, sdpAnswer); + + if (senderName.equals(this.getParticipantPublicId())) { + log.warn("PARTICIPANT {}: trying to configure loopback by subscribing", this.getParticipantPublicId()); + throw new OpenViduException(Code.USER_NOT_STREAMING_ERROR_CODE, "Can loopback only when publishing media"); + } + + KurentoParticipant kSender = (KurentoParticipant) sender; + + if (kSender.streaming && kSender.getPublisher() != null + && kSender.getPublisher().closingLock.readLock().tryLock()) { + + try { + final SubscriberEndpoint subscriber = getSubscriber(senderName); + if (subscriber.getEndpoint() == null) { + throw new OpenViduException(Code.MEDIA_ENDPOINT_ERROR_CODE, "Unable to create subscriber endpoint"); + } + + try { + subscriber.subscribe(sdpAnswer, kSender.getPublisher()); log.info("PARTICIPANT {}: Is now receiving video from {} in room {}", this.getParticipantPublicId(), senderName, this.session.getSessionId()); @@ -279,8 +325,6 @@ public class KurentoParticipant extends Participant { endpointConfig.getCdr().recordNewSubscriber(this, this.session.getSessionId(), sender.getPublisherStreamId(), sender.getParticipantPublicId(), subscriber.createdAt()); } - - return sdpAnswer; } catch (KurentoServerException e) { // TODO Check object status when KurentoClient sets this info in the object if (e.getCode() == 40101) { @@ -292,7 +336,6 @@ public class KurentoParticipant extends Participant { } this.subscribers.remove(senderName); releaseSubscriberEndpoint(senderName, (KurentoParticipant) sender, subscriber, null, false); - return null; } } finally { kSender.getPublisher().closingLock.readLock().unlock(); diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java index cad0d229..681941ea 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java @@ -30,7 +30,6 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import io.openvidu.java.client.*; import org.apache.commons.lang3.RandomStringUtils; import org.kurento.client.GenericMediaElement; import org.kurento.client.IceCandidate; @@ -48,6 +47,12 @@ import com.google.gson.JsonObject; import io.openvidu.client.OpenViduException; import io.openvidu.client.OpenViduException.Code; import io.openvidu.client.internal.ProtocolElements; +import io.openvidu.java.client.MediaMode; +import io.openvidu.java.client.Recording; +import io.openvidu.java.client.RecordingLayout; +import io.openvidu.java.client.RecordingMode; +import io.openvidu.java.client.RecordingProperties; +import io.openvidu.java.client.SessionProperties; import io.openvidu.server.core.EndReason; import io.openvidu.server.core.FinalUser; import io.openvidu.server.core.IdentifierPrefixes; @@ -58,7 +63,6 @@ import io.openvidu.server.core.SessionManager; import io.openvidu.server.core.Token; import io.openvidu.server.kurento.endpoint.KurentoFilter; import io.openvidu.server.kurento.endpoint.PublisherEndpoint; -import io.openvidu.server.kurento.endpoint.SdpType; import io.openvidu.server.kurento.kms.Kms; import io.openvidu.server.kurento.kms.KmsManager; import io.openvidu.server.rpc.RpcHandler; @@ -368,7 +372,6 @@ public class KurentoSessionManager extends SessionManager { kurentoOptions.isOffer, kurentoOptions.sdpOffer, kurentoOptions.doLoopback, kurentoOptions.rtspUri, participant.getParticipantPublicId()); - SdpType sdpType = kurentoOptions.isOffer ? SdpType.OFFER : SdpType.ANSWER; KurentoSession kSession = kParticipant.getSession(); kParticipant.createPublishingEndpoint(mediaOptions, null); @@ -395,7 +398,7 @@ public class KurentoSessionManager extends SessionManager { } } - sdpAnswer = kParticipant.publishToRoom(sdpType, kurentoOptions.sdpOffer, kurentoOptions.doLoopback, false); + sdpAnswer = kParticipant.publishToRoom(kurentoOptions.sdpOffer, kurentoOptions.doLoopback, false); if (sdpAnswer == null) { OpenViduException e = new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, @@ -502,11 +505,54 @@ public class KurentoSessionManager extends SessionManager { } @Override - public void subscribe(Participant participant, String senderName, String sdpOffer, Integer transactionId) { - String sdpAnswer = null; + public void prepareSubscription(Participant participant, String senderPublicId, Integer transactionId) { + String sdpOffer = null; Session session = null; try { - log.debug("Request [SUBSCRIBE] remoteParticipant={} sdpOffer={} ({})", senderName, sdpOffer, + log.debug("Request [SUBSCRIBE] remoteParticipant={} sdpOffer={} ({})", senderPublicId, sdpOffer, + participant.getParticipantPublicId()); + + KurentoParticipant kParticipant = (KurentoParticipant) participant; + session = ((KurentoParticipant) participant).getSession(); + Participant senderParticipant = session.getParticipantByPublicId(senderPublicId); + + if (senderParticipant == null) { + log.warn( + "PARTICIPANT {}: Requesting to recv media from user {} " + + "in session {} but user could not be found", + participant.getParticipantPublicId(), senderPublicId, session.getSessionId()); + throw new OpenViduException(Code.USER_NOT_FOUND_ERROR_CODE, + "User '" + senderPublicId + " not found in session '" + session.getSessionId() + "'"); + } + if (!senderParticipant.isStreaming()) { + log.warn( + "PARTICIPANT {}: Requesting to recv media from user {} " + + "in session {} but user is not streaming media", + participant.getParticipantPublicId(), senderPublicId, session.getSessionId()); + throw new OpenViduException(Code.USER_NOT_STREAMING_ERROR_CODE, + "User '" + senderPublicId + " not streaming media in session '" + session.getSessionId() + "'"); + } + + sdpOffer = kParticipant.prepareReceiveMediaFrom(senderParticipant); + if (sdpOffer == null) { + throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, "Unable to generate SDP offer when subscribing '" + + participant.getParticipantPublicId() + "' to '" + senderPublicId + "'"); + } + } catch (OpenViduException e) { + log.error("PARTICIPANT {}: Error preparing subscription to {}", participant.getParticipantPublicId(), + senderPublicId, e); + sessionEventsHandler.onPrepareSubscription(participant, session, null, transactionId, e); + } + if (sdpOffer != null) { + sessionEventsHandler.onPrepareSubscription(participant, session, sdpOffer, transactionId, null); + } + } + + @Override + public void subscribe(Participant participant, String senderName, String sdpAnswer, Integer transactionId) { + Session session = null; + try { + log.debug("Request [SUBSCRIBE] remoteParticipant={} sdpAnswer={} ({})", senderName, sdpAnswer, participant.getParticipantPublicId()); KurentoParticipant kParticipant = (KurentoParticipant) participant; @@ -530,18 +576,11 @@ public class KurentoSessionManager extends SessionManager { "User '" + senderName + " not streaming media in session '" + session.getSessionId() + "'"); } - sdpAnswer = kParticipant.receiveMediaFrom(senderParticipant, sdpOffer, false); - if (sdpAnswer == null) { - throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, - "Unable to generate SDP answer when subscribing '" + participant.getParticipantPublicId() - + "' to '" + senderName + "'"); - } + kParticipant.receiveMediaFrom(senderParticipant, sdpAnswer, false); + sessionEventsHandler.onSubscribe(participant, session, transactionId, null); } catch (OpenViduException e) { log.error("PARTICIPANT {}: Error subscribing to {}", participant.getParticipantPublicId(), senderName, e); - sessionEventsHandler.onSubscribe(participant, session, null, transactionId, e); - } - if (sdpAnswer != null) { - sessionEventsHandler.onSubscribe(participant, session, sdpAnswer, transactionId, null); + sessionEventsHandler.onSubscribe(participant, session, transactionId, e); } } @@ -1046,7 +1085,7 @@ public class KurentoSessionManager extends SessionManager { } @Override - public void reconnectStream(Participant participant, String streamId, String sdpOffer, Integer transactionId) { + public void reconnectStream(Participant participant, String streamId, String sdpString, Integer transactionId) { KurentoParticipant kParticipant = (KurentoParticipant) participant; KurentoSession kSession = kParticipant.getSession(); @@ -1067,8 +1106,7 @@ public class KurentoSessionManager extends SessionManager { // 3) Create a new PublisherEndpoint connecting it to the previous PassThrough kParticipant.resetPublisherEndpoint(kurentoOptions, passThru); kParticipant.createPublishingEndpoint(kurentoOptions, streamId); - SdpType sdpType = kurentoOptions.isOffer ? SdpType.OFFER : SdpType.ANSWER; - String sdpAnswer = kParticipant.publishToRoom(sdpType, sdpOffer, kurentoOptions.doLoopback, true); + String sdpAnswer = kParticipant.publishToRoom(sdpString, kurentoOptions.doLoopback, true); sessionEventsHandler.onPublishMedia(participant, participant.getPublisherStreamId(), kParticipant.getPublisher().createdAt(), kSession.getSessionId(), kurentoOptions, sdpAnswer, @@ -1081,12 +1119,8 @@ public class KurentoSessionManager extends SessionManager { if (senderPrivateId != null) { KurentoParticipant sender = (KurentoParticipant) kSession.getParticipantByPrivateId(senderPrivateId); kParticipant.cancelReceivingMedia(sender, null, true); - String sdpAnswer = kParticipant.receiveMediaFrom(sender, sdpOffer, true); - if (sdpAnswer == null) { - throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, - "Unable to generate SDP answer when reconnecting subscriber to '" + streamId + "'"); - } - sessionEventsHandler.onSubscribe(participant, kSession, sdpAnswer, transactionId, null); + kParticipant.receiveMediaFrom(sender, sdpString, true); + sessionEventsHandler.onSubscribe(participant, kSession, transactionId, null); } else { throw new OpenViduException(Code.USER_NOT_STREAMING_ERROR_CODE, "Stream '" + streamId + "' does not exist in Session '" + kSession.getSessionId() + "'"); diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/MediaEndpoint.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/MediaEndpoint.java index f4159ed3..d03f42e5 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/MediaEndpoint.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/MediaEndpoint.java @@ -516,6 +516,24 @@ public abstract class MediaEndpoint { } } + protected String generateOffer() throws OpenViduException { + if (this.isWeb()) { + if (webEndpoint == null) { + throw new OpenViduException(Code.MEDIA_WEBRTC_ENDPOINT_ERROR_CODE, + "Can't generate offer when WebRtcEndpoint is null (ep: " + endpointName + ")"); + } + return webEndpoint.generateOffer(); + } else if (this.isPlayerEndpoint()) { + return ""; + } else { + if (endpoint == null) { + throw new OpenViduException(Code.MEDIA_RTP_ENDPOINT_ERROR_CODE, + "Can't generate offer when RtpEndpoint is null (ep: " + endpointName + ")"); + } + return endpoint.generateOffer(); + } + } + /** * If supported, it registers a listener for when a new {@link IceCandidate} is * gathered by the internal endpoint ({@link WebRtcEndpoint}) and sends it to diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java index e35de370..b652cad2 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/PublisherEndpoint.java @@ -175,57 +175,42 @@ public class PublisherEndpoint extends MediaEndpoint { /** * Initializes this media endpoint for publishing media and processes the SDP - * offer or answer. If the internal endpoint is an {@link WebRtcEndpoint}, it - * first registers an event listener for the ICE candidates and instructs the - * endpoint to start gathering the candidates. If required, it connects to - * itself (after applying the intermediate media elements and the - * {@link PassThrough}) to allow loopback of the media stream. + * offer. If the internal endpoint is an {@link WebRtcEndpoint}, it first + * registers an event listener for the ICE candidates and instructs the endpoint + * to start gathering the candidates. If required, it connects to itself (after + * applying the intermediate media elements and the {@link PassThrough}) to + * allow loopback of the media stream. * - * @param sdpType indicates the type of the sdpString (offer or - * answer) - * @param sdpString offer or answer from the remote peer - * @param doLoopback loopback flag - * @param loopbackAlternativeSrc alternative loopback source - * @param loopbackConnectionType how to connect the loopback source - * @return the SDP response (the answer if processing an offer SDP, otherwise is - * the updated offer generated previously by this endpoint) + * @param sdpOffer SDP offer from the remote peer + * @param doLoopback loopback flag + * @return the SDP answer */ - public synchronized String publish(SdpType sdpType, String sdpString, boolean doLoopback) { + public synchronized String publish(String sdpOffer, boolean doLoopback) { + String sdpResponse = processOffer(sdpOffer); registerOnIceCandidateEventListener(this.getOwner().getParticipantPublicId()); if (doLoopback) { - connect(this.getEndpoint()); + connect(this.getEndpoint(), false); } else { - innerConnect(); + innerConnect(false); } this.createdAt = System.currentTimeMillis(); - String sdpResponse = null; - switch (sdpType) { - case ANSWER: - sdpResponse = processAnswer(sdpString); - break; - case OFFER: - sdpResponse = processOffer(sdpString); - break; - default: - throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, "Sdp type not supported: " + sdpType); - } gatherCandidates(); return sdpResponse; } - public synchronized void connect(MediaElement sink) { + public synchronized void connect(MediaElement sink, boolean blocking) { if (!connected) { - innerConnect(); + innerConnect(blocking); } - internalSinkConnect(passThru, sink); + internalSinkConnect(passThru, sink, blocking); this.enableIpCameraIfNecessary(); } - public synchronized void connect(MediaElement sink, MediaType type) { + public synchronized void connect(MediaElement sink, MediaType type, boolean blocking) { if (!connected) { - innerConnect(); + innerConnect(blocking); } - internalSinkConnect(passThru, sink, type); + internalSinkConnect(passThru, sink, type, blocking); this.enableIpCameraIfNecessary(); } @@ -289,11 +274,11 @@ public class PublisherEndpoint extends MediaEndpoint { } if (connected) { if (first != null) { - internalSinkConnect(first, shaper, type); + internalSinkConnect(first, shaper, type, false); } else { - internalSinkConnect(this.getEndpoint(), shaper, type); + internalSinkConnect(this.getEndpoint(), shaper, type, false); } - internalSinkConnect(shaper, passThru, type); + internalSinkConnect(shaper, passThru, type, false); } elementIds.addFirst(id); elements.put(id, shaper); @@ -343,7 +328,7 @@ public class PublisherEndpoint extends MediaEndpoint { } else { prev = passThru; } - internalSinkConnect(next, prev); + internalSinkConnect(next, prev, false); } elementIds.remove(elementId); if (releaseElement) { @@ -408,13 +393,13 @@ public class PublisherEndpoint extends MediaEndpoint { } switch (muteType) { case ALL: - internalSinkConnect(this.getEndpoint(), sink); + internalSinkConnect(this.getEndpoint(), sink, false); break; case AUDIO: - internalSinkConnect(this.getEndpoint(), sink, MediaType.AUDIO); + internalSinkConnect(this.getEndpoint(), sink, MediaType.AUDIO, false); break; case VIDEO: - internalSinkConnect(this.getEndpoint(), sink, MediaType.VIDEO); + internalSinkConnect(this.getEndpoint(), sink, MediaType.VIDEO, false); break; } } @@ -440,7 +425,7 @@ public class PublisherEndpoint extends MediaEndpoint { return elementIds.get(idx - 1); } - private void innerConnect() { + private void innerConnect(boolean blocking) { if (this.getEndpoint() == null) { throw new OpenViduException(Code.MEDIA_ENDPOINT_ERROR_CODE, "Can't connect null endpoint (ep: " + getEndpointName() + ")"); @@ -453,28 +438,32 @@ public class PublisherEndpoint extends MediaEndpoint { throw new OpenViduException(Code.MEDIA_ENDPOINT_ERROR_CODE, "No media element with id " + prevId + " (ep: " + getEndpointName() + ")"); } - internalSinkConnect(current, prev); + internalSinkConnect(current, prev, blocking); current = prev; prevId = getPrevious(prevId); } - internalSinkConnect(current, passThru); + internalSinkConnect(current, passThru, blocking); connected = true; } - private void internalSinkConnect(final MediaElement source, final MediaElement sink) { - source.connect(sink, new Continuation() { - @Override - public void onSuccess(Void result) throws Exception { - log.debug("EP {}: Elements have been connected (source {} -> sink {})", getEndpointName(), - source.getId(), sink.getId()); - } + private void internalSinkConnect(final MediaElement source, final MediaElement sink, boolean blocking) { + if (blocking) { + source.connect(sink); + } else { + source.connect(sink, new Continuation() { + @Override + public void onSuccess(Void result) throws Exception { + log.debug("EP {}: Elements have been connected (source {} -> sink {})", getEndpointName(), + source.getId(), sink.getId()); + } - @Override - public void onError(Throwable cause) throws Exception { - log.warn("EP {}: Failed to connect media elements (source {} -> sink {})", getEndpointName(), - source.getId(), sink.getId(), cause); - } - }); + @Override + public void onError(Throwable cause) throws Exception { + log.warn("EP {}: Failed to connect media elements (source {} -> sink {})", getEndpointName(), + source.getId(), sink.getId(), cause); + } + }); + } } /** @@ -488,23 +477,28 @@ public class PublisherEndpoint extends MediaEndpoint { * be used instead * @see #internalSinkConnect(MediaElement, MediaElement) */ - private void internalSinkConnect(final MediaElement source, final MediaElement sink, final MediaType type) { + private void internalSinkConnect(final MediaElement source, final MediaElement sink, final MediaType type, + boolean blocking) { if (type == null) { - internalSinkConnect(source, sink); + internalSinkConnect(source, sink, blocking); } else { - source.connect(sink, type, new Continuation() { - @Override - public void onSuccess(Void result) throws Exception { - log.debug("EP {}: {} media elements have been connected (source {} -> sink {})", getEndpointName(), - type, source.getId(), sink.getId()); - } + if (blocking) { + source.connect(sink, type); + } else { + source.connect(sink, type, new Continuation() { + @Override + public void onSuccess(Void result) throws Exception { + log.debug("EP {}: {} media elements have been connected (source {} -> sink {})", + getEndpointName(), type, source.getId(), sink.getId()); + } - @Override - public void onError(Throwable cause) throws Exception { - log.warn("EP {}: Failed to connect {} media elements (source {} -> sink {})", getEndpointName(), - type, source.getId(), sink.getId(), cause); - } - }); + @Override + public void onError(Throwable cause) throws Exception { + log.warn("EP {}: Failed to connect {} media elements (source {} -> sink {})", getEndpointName(), + type, source.getId(), sink.getId(), cause); + } + }); + } } } diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/SubscriberEndpoint.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/SubscriberEndpoint.java index db6309e4..ba6eeaca 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/SubscriberEndpoint.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/endpoint/SubscriberEndpoint.java @@ -18,7 +18,6 @@ package io.openvidu.server.kurento.endpoint; import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicBoolean; import org.kurento.client.MediaPipeline; import org.slf4j.Logger; @@ -38,8 +37,6 @@ import io.openvidu.server.kurento.core.KurentoParticipant; public class SubscriberEndpoint extends MediaEndpoint { private final static Logger log = LoggerFactory.getLogger(SubscriberEndpoint.class); - private AtomicBoolean connectedToPublisher = new AtomicBoolean(false); - private String publisherStreamId; public SubscriberEndpoint(EndpointType endpointType, KurentoParticipant owner, String endpointName, @@ -47,23 +44,18 @@ public class SubscriberEndpoint extends MediaEndpoint { super(endpointType, owner, endpointName, pipeline, openviduConfig, log); } - public synchronized String subscribe(String sdpOffer, PublisherEndpoint publisher) { + public synchronized String prepareSubscription(PublisherEndpoint publisher) { registerOnIceCandidateEventListener(publisher.getOwner().getParticipantPublicId()); + publisher.connect(this.getEndpoint(), true); this.createdAt = System.currentTimeMillis(); - String sdpAnswer = processOffer(sdpOffer); - gatherCandidates(); - publisher.connect(this.getEndpoint()); - setConnectedToPublisher(true); this.publisherStreamId = publisher.getStreamId(); - return sdpAnswer; + String sdpOffer = generateOffer(); + gatherCandidates(); + return sdpOffer; } - public boolean isConnectedToPublisher() { - return connectedToPublisher.get(); - } - - public void setConnectedToPublisher(boolean connectedToPublisher) { - this.connectedToPublisher.set(connectedToPublisher); + public synchronized void subscribe(String sdpAnswer, PublisherEndpoint publisher) { + processAnswer(sdpAnswer); } @Override diff --git a/openvidu-server/src/main/java/io/openvidu/server/recording/CompositeWrapper.java b/openvidu-server/src/main/java/io/openvidu/server/recording/CompositeWrapper.java index 7b0b550a..d8946d86 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/recording/CompositeWrapper.java +++ b/openvidu-server/src/main/java/io/openvidu/server/recording/CompositeWrapper.java @@ -111,7 +111,7 @@ public class CompositeWrapper { public void connectPublisherEndpoint(PublisherEndpoint endpoint) throws OpenViduException { HubPort hubPort = new HubPort.Builder(composite).build(); - endpoint.connect(hubPort); + endpoint.connect(hubPort, false); String streamId = endpoint.getOwner().getPublisherStreamId(); this.hubPorts.put(streamId, hubPort); this.publisherEndpoints.put(streamId, endpoint); diff --git a/openvidu-server/src/main/java/io/openvidu/server/recording/service/SingleStreamRecordingService.java b/openvidu-server/src/main/java/io/openvidu/server/recording/service/SingleStreamRecordingService.java index 0e246e34..1691d833 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/recording/service/SingleStreamRecordingService.java +++ b/openvidu-server/src/main/java/io/openvidu/server/recording/service/SingleStreamRecordingService.java @@ -391,14 +391,14 @@ public class SingleStreamRecordingService extends RecordingService { MediaProfileSpecType profile) { switch (profile) { case WEBM: - publisherEndpoint.connect(recorder, MediaType.AUDIO); - publisherEndpoint.connect(recorder, MediaType.VIDEO); + publisherEndpoint.connect(recorder, MediaType.AUDIO, false); + publisherEndpoint.connect(recorder, MediaType.VIDEO, false); break; case WEBM_AUDIO_ONLY: - publisherEndpoint.connect(recorder, MediaType.AUDIO); + publisherEndpoint.connect(recorder, MediaType.AUDIO, false); break; case WEBM_VIDEO_ONLY: - publisherEndpoint.connect(recorder, MediaType.VIDEO); + publisherEndpoint.connect(recorder, MediaType.VIDEO, false); break; default: throw new UnsupportedOperationException("Unsupported profile when single stream recording: " + profile); diff --git a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java index bece2d68..4e742517 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java @@ -126,6 +126,9 @@ public class RpcHandler extends DefaultJsonRpcHandler { case ProtocolElements.ONICECANDIDATE_METHOD: onIceCandidate(rpcConnection, request); break; + case ProtocolElements.PREPARERECEIVEVIDEO_METHOD: + prepareReceiveVideoFrom(rpcConnection, request); + break; case ProtocolElements.RECEIVEVIDEO_METHOD: receiveVideoFrom(rpcConnection, request); break; @@ -333,6 +336,20 @@ public class RpcHandler extends DefaultJsonRpcHandler { } } + private void prepareReceiveVideoFrom(RpcConnection rpcConnection, Request request) { + Participant participant; + try { + participant = sanityCheckOfSession(rpcConnection, "subscribe"); + } catch (OpenViduException e) { + return; + } + + String senderStreamId = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SENDER_PARAM); + String senderPublicId = parseSenderPublicIdFromStreamId(senderStreamId); + + sessionManager.prepareSubscription(participant, senderPublicId, request.getId()); + } + private void receiveVideoFrom(RpcConnection rpcConnection, Request request) { Participant participant; try { @@ -341,23 +358,12 @@ public class RpcHandler extends DefaultJsonRpcHandler { return; } - String senderPublicId = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SENDER_PARAM); + String senderStreamId = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SENDER_PARAM); + String sdpAnswer = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SDPANSWER_PARAM); - // Parse sender public id from stream id - if (senderPublicId.startsWith(IdentifierPrefixes.STREAM_ID + "IPC_") - && senderPublicId.contains(IdentifierPrefixes.IPCAM_ID)) { - // If IPCAM - senderPublicId = senderPublicId.substring(senderPublicId.indexOf("_" + IdentifierPrefixes.IPCAM_ID) + 1, - senderPublicId.length()); - } else { - // Not IPCAM - senderPublicId = senderPublicId.substring( - senderPublicId.lastIndexOf(IdentifierPrefixes.PARTICIPANT_PUBLIC_ID), senderPublicId.length()); - } + String senderPublicId = parseSenderPublicIdFromStreamId(senderStreamId); - String sdpOffer = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SDPOFFER_PARAM); - - sessionManager.subscribe(participant, senderPublicId, sdpOffer, request.getId()); + sessionManager.subscribe(participant, senderPublicId, sdpAnswer, request.getId()); } private void unsubscribeFromVideo(RpcConnection rpcConnection, Request request) { @@ -622,9 +628,9 @@ public class RpcHandler extends DefaultJsonRpcHandler { return; } String streamId = getStringParam(request, ProtocolElements.RECONNECTSTREAM_STREAM_PARAM); - String sdpOffer = getStringParam(request, ProtocolElements.RECONNECTSTREAM_SDPOFFER_PARAM); + String sdpString = getStringParam(request, ProtocolElements.RECONNECTSTREAM_SDPSTRING_PARAM); try { - sessionManager.reconnectStream(participant, streamId, sdpOffer, request.getId()); + sessionManager.reconnectStream(participant, streamId, sdpString, request.getId()); } catch (OpenViduException e) { this.notificationService.sendErrorResponse(participant.getParticipantPrivateId(), request.getId(), new JsonObject(), e); @@ -800,4 +806,20 @@ public class RpcHandler extends DefaultJsonRpcHandler { .equals(this.sessionManager.getParticipantPrivateIdFromStreamId(sessionId, streamId)); } + private String parseSenderPublicIdFromStreamId(String streamId) { + String senderPublicId; + // Parse sender public id from stream id + if (streamId.startsWith(IdentifierPrefixes.STREAM_ID + "IPC_") + && streamId.contains(IdentifierPrefixes.IPCAM_ID)) { + // If IPCAM + senderPublicId = streamId.substring(streamId.indexOf("_" + IdentifierPrefixes.IPCAM_ID) + 1, + streamId.length()); + } else { + // Not IPCAM + senderPublicId = streamId.substring(streamId.lastIndexOf(IdentifierPrefixes.PARTICIPANT_PUBLIC_ID), + streamId.length()); + } + return senderPublicId; + } + } From 4ea531a379388f2d0cd9a0178756ee449df0f2ac Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Wed, 2 Sep 2020 13:25:38 +0200 Subject: [PATCH 3/7] openvidu-server: support reconnection with mediasoup --- .../io/openvidu/client/internal/ProtocolElements.java | 1 + .../main/java/io/openvidu/server/core/SessionManager.java | 2 +- .../server/kurento/core/KurentoSessionManager.java | 8 ++++++-- .../src/main/java/io/openvidu/server/rpc/RpcHandler.java | 7 ++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index 5e4d62c9..676489d8 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -72,6 +72,7 @@ public class ProtocolElements { public static final String PREPARERECEIVEVIDEO_METHOD = "prepareReceiveVideFrom"; public static final String PREPARERECEIVEVIDEO_SDPOFFER_PARAM = "sdpOffer"; + public static final String PREPARERECEIVEVIDEO_RECONNECT_PARAM = "reconnect"; public static final String RECEIVEVIDEO_METHOD = "receiveVideoFrom"; public static final String RECEIVEVIDEO_SDPOFFER_PARAM = "sdpOffer"; diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java index af55da6e..9658dfac 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java @@ -106,7 +106,7 @@ public abstract class SessionManager { public abstract void unpublishVideo(Participant participant, Participant moderator, Integer transactionId, EndReason reason); - public abstract void prepareSubscription(Participant participant, String senderPublicId, Integer id); + public abstract void prepareSubscription(Participant participant, String senderPublicId, boolean reconnect, Integer id); public abstract void subscribe(Participant participant, String senderName, String sdpAnwser, Integer transactionId); diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java index 681941ea..e9d4a63e 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java @@ -505,7 +505,8 @@ public class KurentoSessionManager extends SessionManager { } @Override - public void prepareSubscription(Participant participant, String senderPublicId, Integer transactionId) { + public void prepareSubscription(Participant participant, String senderPublicId, boolean reconnect, + Integer transactionId) { String sdpOffer = null; Session session = null; try { @@ -533,6 +534,10 @@ public class KurentoSessionManager extends SessionManager { "User '" + senderPublicId + " not streaming media in session '" + session.getSessionId() + "'"); } + if (reconnect) { + kParticipant.cancelReceivingMedia(((KurentoParticipant) senderParticipant), null, true); + } + sdpOffer = kParticipant.prepareReceiveMediaFrom(senderParticipant); if (sdpOffer == null) { throw new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, "Unable to generate SDP offer when subscribing '" @@ -1118,7 +1123,6 @@ public class KurentoSessionManager extends SessionManager { String senderPrivateId = kSession.getParticipantPrivateIdFromStreamId(streamId); if (senderPrivateId != null) { KurentoParticipant sender = (KurentoParticipant) kSession.getParticipantByPrivateId(senderPrivateId); - kParticipant.cancelReceivingMedia(sender, null, true); kParticipant.receiveMediaFrom(sender, sdpString, true); sessionEventsHandler.onSubscribe(participant, kSession, transactionId, null); } else { diff --git a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java index 4e742517..42f5b57a 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java @@ -339,21 +339,22 @@ public class RpcHandler extends DefaultJsonRpcHandler { private void prepareReceiveVideoFrom(RpcConnection rpcConnection, Request request) { Participant participant; try { - participant = sanityCheckOfSession(rpcConnection, "subscribe"); + participant = sanityCheckOfSession(rpcConnection, "prepareReceiveVideFrom"); } catch (OpenViduException e) { return; } String senderStreamId = getStringParam(request, ProtocolElements.RECEIVEVIDEO_SENDER_PARAM); String senderPublicId = parseSenderPublicIdFromStreamId(senderStreamId); + boolean reconnect = getBooleanParam(request, ProtocolElements.PREPARERECEIVEVIDEO_RECONNECT_PARAM); - sessionManager.prepareSubscription(participant, senderPublicId, request.getId()); + sessionManager.prepareSubscription(participant, senderPublicId, reconnect, request.getId()); } private void receiveVideoFrom(RpcConnection rpcConnection, Request request) { Participant participant; try { - participant = sanityCheckOfSession(rpcConnection, "subscribe"); + participant = sanityCheckOfSession(rpcConnection, "receiveVideoFrom"); } catch (OpenViduException e) { return; } From 23d64be8063f8fdb2a212ca845e304762f2803f5 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Wed, 2 Sep 2020 13:29:25 +0200 Subject: [PATCH 4/7] openvidu-browser: removed Ionic iOS timeout on first subscription --- openvidu-browser/src/OpenVidu/Session.ts | 19 --------------- openvidu-browser/src/OpenVidu/Stream.ts | 20 ---------------- .../OpenViduInternal/WebRtcPeer/WebRtcPeer.ts | 23 ++++--------------- 3 files changed, 4 insertions(+), 58 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/Session.ts b/openvidu-browser/src/OpenVidu/Session.ts index 61f50bc9..a02f8c1c 100644 --- a/openvidu-browser/src/OpenVidu/Session.ts +++ b/openvidu-browser/src/OpenVidu/Session.ts @@ -99,14 +99,6 @@ export class Session extends EventDispatcher { */ remoteStreamsCreated: ObjMap = {}; - /** - * @hidden - */ - isFirstIonicIosSubscriber = true; - /** - * @hidden - */ - countDownForIonicIosSubscribersActive = true; /** * @hidden */ @@ -724,11 +716,6 @@ export class Session extends EventDispatcher { streamEvent.callDefaultBehavior(); delete this.remoteStreamsCreated[stream.streamId]; - - if (Object.keys(this.remoteStreamsCreated).length === 0) { - this.isFirstIonicIosSubscriber = true; - this.countDownForIonicIosSubscribersActive = true; - } } delete this.remoteConnections[connection.connectionId]; this.ee.emitEvent('connectionDestroyed', [new ConnectionEvent(false, this, 'connectionDestroyed', connection, msg.reason)]); @@ -798,12 +785,6 @@ export class Session extends EventDispatcher { // Deleting the remote stream const streamId: string = connection.stream.streamId; delete this.remoteStreamsCreated[streamId]; - - if (Object.keys(this.remoteStreamsCreated).length === 0) { - this.isFirstIonicIosSubscriber = true; - this.countDownForIonicIosSubscribersActive = true; - } - connection.removeStream(streamId); }) .catch(openViduError => { diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index 9178f3af..d973a438 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -965,26 +965,6 @@ export class Stream extends EventDispatcher { reject(new Error('Error on receiveVideFrom: ' + JSON.stringify(error))); } else { resolve(); - // // Ios Ionic. Limitation: some bug in iosrtc cordova plugin makes it necessary - // // to add a timeout before calling PeerConnection#setRemoteDescription during - // // some time (400 ms) from the moment first subscriber stream is received - // if (this.session.isFirstIonicIosSubscriber) { - // this.session.isFirstIonicIosSubscriber = false; - // setTimeout(() => { - // // After 400 ms Ionic iOS subscribers won't need to run - // // PeerConnection#setRemoteDescription after 250 ms timeout anymore - // this.session.countDownForIonicIosSubscribersActive = false; - // }, 400); - // } - // const needsTimeoutOnProcessAnswer = this.session.countDownForIonicIosSubscribersActive; - // this.webRtcPeer.processAnswer(response.sdpAnswer, needsTimeoutOnProcessAnswer).then(() => { - // logger.info("'Subscriber' (" + this.streamId + ") successfully " + (reconnect ? "reconnected" : "subscribed")); - // this.remotePeerSuccessfullyEstablished(); - // this.initWebRtcStats(); - // resolve(); - // }).catch(error => { - // reject(error); - // }); } }); }; diff --git a/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts b/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts index c4141973..b36488d2 100644 --- a/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts +++ b/openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts @@ -232,8 +232,7 @@ export class WebRtcPeer { if (this.pc.signalingState === 'closed') { reject('RTCPeerConnection is closed when trying to set remote description'); } - // TODO: check if Ionic iOS still needs timeout on setting first remote description when subscribing - this.setRemoteDescription(offer, false) + this.setRemoteDescription(offer) .then(() => { resolve(); }) @@ -272,7 +271,7 @@ export class WebRtcPeer { if (this.pc.signalingState === 'closed') { reject('RTCPeerConnection is closed when trying to set remote description'); } - this.setRemoteDescription(answer, false) + this.setRemoteDescription(answer) .then(() => resolve()) .catch(error => reject(error)); }); @@ -281,22 +280,8 @@ export class WebRtcPeer { /** * @hidden */ - async setRemoteDescription(sdp: RTCSessionDescriptionInit, needsTimeoutOnProcessAnswer: boolean): Promise { - // if (platform['isIonicIos']) { - // // Ionic iOS platform - // if (needsTimeoutOnProcessAnswer) { - // // 400 ms have not elapsed yet since first remote stream triggered Stream#initWebRtcPeerReceive - // await new Promise(resolve => setTimeout(resolve, 250)); // Sleep for 250ms - // logger.info('setRemoteDescription run after timeout for Ionic iOS device'); - // return this.pc.setRemoteDescription(sdp); - // } else { - // // 400 ms have elapsed - // return this.pc.setRemoteDescription(sdp); - // } - // } else { - // Rest of platforms - return this.pc.setRemoteDescription(sdp); - // } + async setRemoteDescription(sdp: RTCSessionDescriptionInit): Promise { + return this.pc.setRemoteDescription(sdp); } /** From d2ce4c297ca89acbeb45c5cf822f6c66af803f7d Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Wed, 2 Sep 2020 13:30:35 +0200 Subject: [PATCH 5/7] openvidu-browser: support reconnection with mediasoup --- openvidu-browser/src/OpenVidu/Stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index d973a438..832a03e1 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -913,7 +913,7 @@ export class Stream extends EventDispatcher { */ initWebRtcPeerReceive(reconnect: boolean): Promise { return new Promise((resolve, reject) => { - this.session.openvidu.sendRequest('prepareReceiveVideFrom', { sender: this.streamId }, (error, response) => { + this.session.openvidu.sendRequest('prepareReceiveVideFrom', { sender: this.streamId, reconnect }, (error, response) => { if (error) { reject(new Error('Error on prepareReceiveVideFrom: ' + JSON.stringify(error))); } else { From 42a2fed910442be6f72f947b824531e11244da44 Mon Sep 17 00:00:00 2001 From: cruizba Date: Thu, 3 Sep 2020 19:39:43 +0200 Subject: [PATCH 6/7] Force codec to avoid transcoding working. --- .../io/openvidu/client/OpenViduException.java | 4 +- openvidu-java-client/generate-docs.sh | 4 +- .../java/io/openvidu/java/client/Session.java | 21 ++- .../java/client/SessionProperties.java | 62 +++++++- .../io/openvidu/java/client/VideoCodec.java | 25 ++++ openvidu-node-client/src/Session.ts | 12 +- openvidu-node-client/src/SessionProperties.ts | 20 +++ openvidu-node-client/src/VideoCodec.ts | 10 ++ openvidu-node-client/src/index.ts | 3 +- .../deployments/ce/docker-compose/.env | 2 +- .../pro/docker-compose/media-node/.env | 2 +- .../io/openvidu/server/OpenViduServer.java | 7 + .../java/io/openvidu/server/core/Session.java | 8 +- .../openvidu/server/core/SessionManager.java | 10 +- .../kurento/core/KurentoSessionManager.java | 74 ++++++++-- .../server/rest/SessionRestController.java | 27 +++- .../io/openvidu/server/rpc/RpcHandler.java | 10 +- .../io/openvidu/server/utils/SDPMunging.java | 133 ++++++++++++++++++ .../session-properties-dialog.component.css | 4 + .../session-properties-dialog.component.html | 12 ++ .../session-properties-dialog.component.ts | 3 +- .../openvidu-instance.component.ts | 7 +- 22 files changed, 414 insertions(+), 46 deletions(-) create mode 100644 openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java create mode 100644 openvidu-node-client/src/VideoCodec.ts create mode 100644 openvidu-server/src/main/java/io/openvidu/server/utils/SDPMunging.java diff --git a/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java b/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java index c6757c23..c2c47515 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java +++ b/openvidu-client/src/main/java/io/openvidu/client/OpenViduException.java @@ -49,7 +49,9 @@ public class OpenViduException extends JsonRpcErrorException { DOCKER_NOT_FOUND(709), RECORDING_PATH_NOT_VALID(708), RECORDING_FILE_EMPTY_ERROR(707), RECORDING_DELETE_ERROR_CODE(706), RECORDING_LIST_ERROR_CODE(705), RECORDING_STOP_ERROR_CODE(704), - RECORDING_START_ERROR_CODE(703), RECORDING_REPORT_ERROR_CODE(702), RECORDING_COMPLETION_ERROR_CODE(701); + RECORDING_START_ERROR_CODE(703), RECORDING_REPORT_ERROR_CODE(702), RECORDING_COMPLETION_ERROR_CODE(701), + + FORCED_CODEC_NOT_FOUND_IN_SDPOFFER(800); private int value; diff --git a/openvidu-java-client/generate-docs.sh b/openvidu-java-client/generate-docs.sh index 3abb098a..e686c5fe 100755 --- a/openvidu-java-client/generate-docs.sh +++ b/openvidu-java-client/generate-docs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x if [[ -z "$BASEHREF_VERSION" ]]; then echo "Example of use: \"BASEHREF_VERSION=2.12.0 ${0}\"" 1>&2 @@ -9,7 +9,7 @@ fi grep -rl '/en/stable/' ./src | xargs sed -i -e 's|/en/stable/|/en/'${BASEHREF_VERSION}'/|g' # Generate JavaDoc -mvn javadoc:javadoc +mvn javadoc:javadoc -DadditionalJOption=-Xdoclint:none rm -rf ../../openvidu.io/api/openvidu-java-client/* cp -R ./target/site/apidocs/. ../../openvidu.io-docs/docs/api/openvidu-java-client diff --git a/openvidu-java-client/src/main/java/io/openvidu/java/client/Session.java b/openvidu-java-client/src/main/java/io/openvidu/java/client/Session.java index c2e3bcc4..8b7da5d8 100644 --- a/openvidu-java-client/src/main/java/io/openvidu/java/client/Session.java +++ b/openvidu-java-client/src/main/java/io/openvidu/java/client/Session.java @@ -25,6 +25,11 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; + import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -35,11 +40,6 @@ import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; - public class Session { private static final Logger log = LoggerFactory.getLogger(Session.class); @@ -458,6 +458,8 @@ public class Session { json.addProperty("defaultRecordingLayout", properties.defaultRecordingLayout().name()); json.addProperty("defaultCustomLayout", properties.defaultCustomLayout()); json.addProperty("customSessionId", properties.customSessionId()); + json.addProperty("forcedVideoCodec", properties.forcedVideoCodec().name()); + json.addProperty("allowTranscoding", properties.isTranscodingAllowed()); StringEntity params = null; try { params = new StringEntity(json.toString()); @@ -520,6 +522,12 @@ public class Session { if (json.has("defaultCustomLayout")) { builder.defaultCustomLayout(json.get("defaultCustomLayout").getAsString()); } + if (json.has("forcedVideoCodec")) { + builder.forcedVideoCodec(VideoCodec.valueOf(json.get("forcedVideoCodec").getAsString())); + } + if (json.has("allowTranscoding")) { + builder.allowTranscoding(json.get("allowTranscoding").getAsBoolean()); + } if (this.properties != null && this.properties.customSessionId() != null) { builder.customSessionId(this.properties.customSessionId()); } else if (json.has("customSessionId")) { @@ -572,6 +580,9 @@ public class Session { json.addProperty("defaultOutputMode", this.properties.defaultOutputMode().name()); json.addProperty("defaultRecordingLayout", this.properties.defaultRecordingLayout().name()); json.addProperty("defaultCustomLayout", this.properties.defaultCustomLayout()); + json.addProperty("forcedVideoCodec", this.properties.forcedVideoCodec().name()); + json.addProperty("allowTranscoding", this.properties.isTranscodingAllowed()); + JsonObject connections = new JsonObject(); connections.addProperty("numberOfElements", this.getActiveConnections().size()); JsonArray jsonArrayConnections = new JsonArray(); diff --git a/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java b/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java index 8f436721..59842409 100644 --- a/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java +++ b/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java @@ -30,6 +30,8 @@ public class SessionProperties { private RecordingLayout defaultRecordingLayout; private String defaultCustomLayout; private String customSessionId; + private VideoCodec forcedVideoCodec; + private boolean allowTranscoding; /** * Builder for {@link io.openvidu.java.client.SessionProperties} @@ -42,6 +44,8 @@ public class SessionProperties { private RecordingLayout defaultRecordingLayout = RecordingLayout.BEST_FIT; private String defaultCustomLayout = ""; private String customSessionId = ""; + private VideoCodec forcedVideoCodec = VideoCodec.VP8; + private boolean allowTranscoding = false; /** * Returns the {@link io.openvidu.java.client.SessionProperties} object properly @@ -49,7 +53,8 @@ public class SessionProperties { */ public SessionProperties build() { return new SessionProperties(this.mediaMode, this.recordingMode, this.defaultOutputMode, - this.defaultRecordingLayout, this.defaultCustomLayout, this.customSessionId); + this.defaultRecordingLayout, this.defaultCustomLayout, this.customSessionId, + this.forcedVideoCodec, this.allowTranscoding); } /** @@ -137,6 +142,34 @@ public class SessionProperties { this.customSessionId = customSessionId; return this; } + + /** + * + * Call this method to define which video codec do you want to be forcibly used for this session. + * This allows browsers to use the same codec avoiding transcoding in the media server. + * To force this video codec you need to set {@link #allowTranscoding(boolean)} to false. + * + */ + public SessionProperties.Builder forcedVideoCodec(VideoCodec forcedVideoCodec) { + this.forcedVideoCodec = forcedVideoCodec; + return this; + } + + /** + * + * Call this method to define if you want to allowTranscoding or not. If you define it as + * as false, the default video codec VP8 will be used for all browsers, and the media + * server will not do any transcoding. If you define it as true, transcoding can be + * executed by the media server when necessary. + * + * If you want to set a different video codec, you can configure it + * by calling {@link #forcedVideoCodec(VideoCodec)} to your preferred one. + * + */ + public SessionProperties.Builder allowTranscoding(boolean allowTranscoding) { + this.allowTranscoding = allowTranscoding; + return this; + } } @@ -147,16 +180,21 @@ public class SessionProperties { this.defaultRecordingLayout = RecordingLayout.BEST_FIT; this.defaultCustomLayout = ""; this.customSessionId = ""; + this.forcedVideoCodec = VideoCodec.VP8; + this.allowTranscoding = false; } private SessionProperties(MediaMode mediaMode, RecordingMode recordingMode, OutputMode outputMode, - RecordingLayout layout, String defaultCustomLayout, String customSessionId) { + RecordingLayout layout, String defaultCustomLayout, String customSessionId, + VideoCodec forcedVideoCodec, boolean allowTranscoding) { this.mediaMode = mediaMode; this.recordingMode = recordingMode; this.defaultOutputMode = outputMode; this.defaultRecordingLayout = layout; this.defaultCustomLayout = defaultCustomLayout; this.customSessionId = customSessionId; + this.forcedVideoCodec = forcedVideoCodec; + this.allowTranscoding = allowTranscoding; } /** @@ -230,5 +268,25 @@ public class SessionProperties { public String customSessionId() { return this.customSessionId; } + + /** + * + * Defines which video codec is being forced to be used when + * {@link io.openvidu.java.client.SessionProperties.Builder#allowTranscoding(boolean)} + * has been set to false + */ + public VideoCodec forcedVideoCodec() { + return this.forcedVideoCodec; + } + + /** + * + * Defines if transcoding is allowed or not. If this method returns false, a video codec + * will be forcibly used for all browsers (See + * {@link io.openvidu.java.client.SessionProperties.Builder#forcedVideoCodec(VideoCodec)}). + */ + public boolean isTranscodingAllowed() { + return this.allowTranscoding; + } } \ No newline at end of file diff --git a/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java b/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java new file mode 100644 index 00000000..9f6e002f --- /dev/null +++ b/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java @@ -0,0 +1,25 @@ +/* + * (C) Copyright 2017-2020 OpenVidu (https://openvidu.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.openvidu.java.client; + +/** + * See {@link io.openvidu.java.client.SessionProperties.Builder#forcedVideoCodec(VideoCodec)} + */ +public enum VideoCodec { + VP8, VP9, H264 +} \ No newline at end of file diff --git a/openvidu-node-client/src/Session.ts b/openvidu-node-client/src/Session.ts index a9fead7b..3b5b4486 100644 --- a/openvidu-node-client/src/Session.ts +++ b/openvidu-node-client/src/Session.ts @@ -26,6 +26,7 @@ import { RecordingLayout } from './RecordingLayout'; import { RecordingMode } from './RecordingMode'; import { SessionProperties } from './SessionProperties'; import { TokenOptions } from './TokenOptions'; +import { VideoCodec } from './VideoCodec'; export class Session { @@ -83,6 +84,8 @@ export class Session { this.properties.recordingMode = !!this.properties.recordingMode ? this.properties.recordingMode : RecordingMode.MANUAL; this.properties.defaultOutputMode = !!this.properties.defaultOutputMode ? this.properties.defaultOutputMode : Recording.OutputMode.COMPOSED; this.properties.defaultRecordingLayout = !!this.properties.defaultRecordingLayout ? this.properties.defaultRecordingLayout : RecordingLayout.BEST_FIT; + this.properties.allowTranscoding = !!this.properties.allowTranscoding ? this.properties.allowTranscoding : false; + this.properties.forcedVideoCodec = !!this.properties.forcedVideoCodec ? this.properties.forcedVideoCodec : VideoCodec.VP8; } /** @@ -401,7 +404,10 @@ export class Session { defaultOutputMode: !!this.properties.defaultOutputMode ? this.properties.defaultOutputMode : Recording.OutputMode.COMPOSED, defaultRecordingLayout: !!this.properties.defaultRecordingLayout ? this.properties.defaultRecordingLayout : RecordingLayout.BEST_FIT, defaultCustomLayout: !!this.properties.defaultCustomLayout ? this.properties.defaultCustomLayout : '', - customSessionId: !!this.properties.customSessionId ? this.properties.customSessionId : '' + customSessionId: !!this.properties.customSessionId ? this.properties.customSessionId : '', + forcedVideoCodec: !!this.properties.forcedVideoCodec ? this.properties.forcedVideoCodec : VideoCodec.VP8, + allowTranscoding: !!this.properties.allowTranscoding ? this.properties.allowTranscoding : false + }); axios.post( @@ -466,7 +472,9 @@ export class Session { mediaMode: json.mediaMode, recordingMode: json.recordingMode, defaultOutputMode: json.defaultOutputMode, - defaultRecordingLayout: json.defaultRecordingLayout + defaultRecordingLayout: json.defaultRecordingLayout, + forcedVideoCodec: !!json.forcedVideoCodec ? json.forcedVideoCodec : VideoCodec.VP8, + allowTranscoding: !!json.allowTranscoding ? json.allowTranscoding : false }; if (!!customSessionId) { this.properties.customSessionId = customSessionId; diff --git a/openvidu-node-client/src/SessionProperties.ts b/openvidu-node-client/src/SessionProperties.ts index f866e1b8..df45ac41 100644 --- a/openvidu-node-client/src/SessionProperties.ts +++ b/openvidu-node-client/src/SessionProperties.ts @@ -64,4 +64,24 @@ export interface SessionProperties { * If this parameter is undefined or an empty string, OpenVidu Server will generate a random sessionId for you. */ customSessionId?: string; + + /** + * Call this method to define which video codec do you want to be forcibly used for this session. + * This allows browsers to use the same codec avoiding transcoding in the media server. + * To force this video codec you need to set [[allowTranscoding]] to false. + */ + forcedVideoCodec?: string; + + /** + * Call this method to define if you want to allowTranscoding or not. If you define it as + * as false, the default video codec VP8 will be used for all browsers, and the media + * server will not do any transcoding. If you define it as true, transcoding can be + * executed by the media server when necessary. + * + * If you want to set a different video codec, you can configure it + * by calling [[forcedVideoCodec]] to your preferred one. + * + */ + allowTranscoding?: boolean; + } diff --git a/openvidu-node-client/src/VideoCodec.ts b/openvidu-node-client/src/VideoCodec.ts new file mode 100644 index 00000000..e1077f77 --- /dev/null +++ b/openvidu-node-client/src/VideoCodec.ts @@ -0,0 +1,10 @@ +/** + * See [[SessionProperties.forcedVideoCodec]] + */ +export enum VideoCodec { + + VP8 = 'VP8', + VP9 = 'VP9', + H264 = 'H264' + +} \ No newline at end of file diff --git a/openvidu-node-client/src/index.ts b/openvidu-node-client/src/index.ts index 50ce477a..4113ac84 100644 --- a/openvidu-node-client/src/index.ts +++ b/openvidu-node-client/src/index.ts @@ -9,4 +9,5 @@ export * from './RecordingMode'; export * from './Recording'; export * from './RecordingProperties'; export * from './Connection'; -export * from './Publisher'; \ No newline at end of file +export * from './Publisher'; +export * from './VideoCodec'; \ No newline at end of file diff --git a/openvidu-server/deployments/ce/docker-compose/.env b/openvidu-server/deployments/ce/docker-compose/.env index 279ec85e..29ac0cb5 100644 --- a/openvidu-server/deployments/ce/docker-compose/.env +++ b/openvidu-server/deployments/ce/docker-compose/.env @@ -149,7 +149,7 @@ OPENVIDU_CDR_PATH=/opt/openvidu/cdr # -------------------------- # Docker hub kurento media server: https://hub.docker.com/r/kurento/kurento-media-server-dev # Uncomment the next line and define this variable with KMS image that you want use -# KMS_IMAGE=kurento/kurento-media-server-dev:6.14.0 +# KMS_IMAGE=kurento/kurento-media-server:6.14.0 # Kurento Media Server Level logs # ------------------------------- diff --git a/openvidu-server/deployments/pro/docker-compose/media-node/.env b/openvidu-server/deployments/pro/docker-compose/media-node/.env index 704c9211..8108613b 100644 --- a/openvidu-server/deployments/pro/docker-compose/media-node/.env +++ b/openvidu-server/deployments/pro/docker-compose/media-node/.env @@ -8,7 +8,7 @@ # -------------------------- # Docker hub kurento media server: https://hub.docker.com/r/kurento/kurento-media-server-dev # Uncomment the next line and define this variable with KMS image that you want use -# KMS_IMAGE=kurento/kurento-media-server-dev:6.14.0 +# KMS_IMAGE=kurento/kurento-media-server:6.14.0 # Kurento Media Server Level logs # ------------------------------- diff --git a/openvidu-server/src/main/java/io/openvidu/server/OpenViduServer.java b/openvidu-server/src/main/java/io/openvidu/server/OpenViduServer.java index ee2d8cbe..a81943f1 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/OpenViduServer.java +++ b/openvidu-server/src/main/java/io/openvidu/server/OpenViduServer.java @@ -72,6 +72,7 @@ import io.openvidu.server.utils.MediaNodeStatusManager; import io.openvidu.server.utils.MediaNodeStatusManagerDummy; import io.openvidu.server.utils.QuarantineKiller; import io.openvidu.server.utils.QuarantineKillerDummy; +import io.openvidu.server.utils.SDPMunging; import io.openvidu.server.webhook.CDRLoggerWebhook; /** @@ -195,6 +196,12 @@ public class OpenViduServer implements JsonRpcConfigurer { public GeoLocationByIp geoLocationByIp() { return new GeoLocationByIpDummy(); } + + @Bean + @ConditionalOnMissingBean + public SDPMunging sdpMunging() { + return new SDPMunging(); + } @Bean @ConditionalOnMissingBean diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/Session.java b/openvidu-server/src/main/java/io/openvidu/server/core/Session.java index 7edf339c..bf0c0807 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/Session.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/Session.java @@ -29,12 +29,12 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import io.openvidu.client.OpenViduException; import io.openvidu.client.OpenViduException.Code; import io.openvidu.client.internal.ProtocolElements; @@ -200,6 +200,8 @@ public class Session implements SessionInterface { json.addProperty("mediaMode", this.sessionProperties.mediaMode().name()); json.addProperty("recordingMode", this.sessionProperties.recordingMode().name()); json.addProperty("defaultOutputMode", this.sessionProperties.defaultOutputMode().name()); + json.addProperty("forcedVideoCodec", this.sessionProperties.forcedVideoCodec().name()); + json.addProperty("allowTranscoding", this.sessionProperties.isTranscodingAllowed()); if (RecordingUtils.IS_COMPOSED(this.sessionProperties.defaultOutputMode())) { json.addProperty("defaultRecordingLayout", this.sessionProperties.defaultRecordingLayout().name()); if (RecordingLayout.CUSTOM.equals(this.sessionProperties.defaultRecordingLayout())) { diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java index af55da6e..a2e0411b 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionManager.java @@ -32,17 +32,17 @@ import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; + import org.apache.commons.lang3.RandomStringUtils; import org.kurento.jsonrpc.message.Request; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; - import io.openvidu.client.OpenViduException; import io.openvidu.client.OpenViduException.Code; import io.openvidu.client.internal.ProtocolElements; diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java index 681941ea..e546ff5e 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java @@ -30,6 +30,9 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + import org.apache.commons.lang3.RandomStringUtils; import org.kurento.client.GenericMediaElement; import org.kurento.client.IceCandidate; @@ -41,9 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - import io.openvidu.client.OpenViduException; import io.openvidu.client.OpenViduException.Code; import io.openvidu.client.internal.ProtocolElements; @@ -53,6 +53,7 @@ import io.openvidu.java.client.RecordingLayout; import io.openvidu.java.client.RecordingMode; import io.openvidu.java.client.RecordingProperties; import io.openvidu.java.client.SessionProperties; +import io.openvidu.java.client.VideoCodec; import io.openvidu.server.core.EndReason; import io.openvidu.server.core.FinalUser; import io.openvidu.server.core.IdentifierPrefixes; @@ -68,6 +69,7 @@ import io.openvidu.server.kurento.kms.KmsManager; import io.openvidu.server.rpc.RpcHandler; import io.openvidu.server.utils.GeoLocation; import io.openvidu.server.utils.JsonUtils; +import io.openvidu.server.utils.SDPMunging; public class KurentoSessionManager extends SessionManager { @@ -81,6 +83,9 @@ public class KurentoSessionManager extends SessionManager { @Autowired private KurentoParticipantEndpointConfig kurentoEndpointConfig; + + @Autowired + private SDPMunging sdpMunging; @Override /* Protected by Session.closingLock.readLock */ @@ -365,15 +370,29 @@ public class KurentoSessionManager extends SessionManager { KurentoMediaOptions kurentoOptions = (KurentoMediaOptions) mediaOptions; KurentoParticipant kParticipant = (KurentoParticipant) participant; - + KurentoSession kSession = kParticipant.getSession(); + + // Modify sdp if transcoding is not allowed + if(!kSession.getSessionProperties().isTranscodingAllowed()) { + VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + String sdpOffer = kurentoOptions.sdpOffer; + + try { + kurentoOptions.sdpOffer = modifySdpToForceCodec(forcedVideoCodec, sdpOffer); + } catch (OpenViduException e) { + String errorMessage = "Error forcing codec: ''" + forcedVideoCodec + "', for publisher on Session: '" + kSession.getSessionId() + + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpOffer; + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + + } + log.debug( "Request [PUBLISH_MEDIA] isOffer={} sdp={} " + "loopbackAltSrc={} lpbkConnType={} doLoopback={} rtspUri={} ({})", kurentoOptions.isOffer, kurentoOptions.sdpOffer, kurentoOptions.doLoopback, kurentoOptions.rtspUri, participant.getParticipantPublicId()); - KurentoSession kSession = kParticipant.getSession(); - kParticipant.createPublishingEndpoint(mediaOptions, null); /* @@ -397,7 +416,7 @@ public class KurentoSessionManager extends SessionManager { throw e; } } - + sdpAnswer = kParticipant.publishToRoom(kurentoOptions.sdpOffer, kurentoOptions.doLoopback, false); if (sdpAnswer == null) { @@ -558,6 +577,18 @@ public class KurentoSessionManager extends SessionManager { KurentoParticipant kParticipant = (KurentoParticipant) participant; session = ((KurentoParticipant) participant).getSession(); Participant senderParticipant = session.getParticipantByPublicId(senderName); + + // Modify sdp if transcoding is not allowed + if (!session.getSessionProperties().isTranscodingAllowed()) { + VideoCodec forcedVideoCodec = session.getSessionProperties().forcedVideoCodec(); + try { + sdpAnswer = this.modifySdpToForceCodec(forcedVideoCodec, sdpAnswer); + } catch (OpenViduException e) { + String errorMessage = "Error forcing codec: ''" + forcedVideoCodec + "', for subscriber on Session: '" + + session.getSessionId() + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpAnswer; + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + } if (senderParticipant == null) { log.warn( @@ -1086,10 +1117,26 @@ public class KurentoSessionManager extends SessionManager { @Override public void reconnectStream(Participant participant, String streamId, String sdpString, Integer transactionId) { + KurentoParticipant kParticipant = (KurentoParticipant) participant; KurentoSession kSession = kParticipant.getSession(); + boolean isPublisher = streamId.equals(participant.getPublisherStreamId()); + + // Modify sdp if transcoding is not allowed + if (!kSession.getSessionProperties().isTranscodingAllowed()) { + VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + try { + sdpString = modifySdpToForceCodec(forcedVideoCodec, sdpString); + } catch (OpenViduException e) { + String errorMessage = "Error on reconnecting and forcing codec: ''" + forcedVideoCodec + "', for " + + (isPublisher ? "publisher" : "subscriber") + " on Session: '" + kSession.getSessionId() + + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpString; + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + + } - if (streamId.equals(participant.getPublisherStreamId())) { + if (isPublisher) { // Reconnect publisher final KurentoMediaOptions kurentoOptions = (KurentoMediaOptions) kParticipant.getPublisher() @@ -1192,5 +1239,14 @@ public class KurentoSessionManager extends SessionManager { filter.removeEventListener(pub.removeListener(eventType)); } } - + + + private String modifySdpToForceCodec(VideoCodec codec, String sdpOffer) { + // Modify sdpOffer if transcoding is not allowed + String modSdpOffer = this.sdpMunging.setCodecPreference(codec, sdpOffer); + if (modSdpOffer != null) { + sdpOffer = modSdpOffer; + } + return sdpOffer; + } } diff --git a/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java b/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java index 28da7b54..e773c314 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java @@ -24,6 +24,12 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,12 +46,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - import io.openvidu.client.OpenViduException; import io.openvidu.client.internal.ProtocolElements; import io.openvidu.java.client.MediaMode; @@ -55,6 +55,7 @@ import io.openvidu.java.client.RecordingLayout; import io.openvidu.java.client.RecordingMode; import io.openvidu.java.client.RecordingProperties; import io.openvidu.java.client.SessionProperties; +import io.openvidu.java.client.VideoCodec; import io.openvidu.server.config.OpenviduConfig; import io.openvidu.server.core.EndReason; import io.openvidu.server.core.IdentifierPrefixes; @@ -102,6 +103,8 @@ public class SessionRestController { String defaultOutputModeString; String defaultRecordingLayoutString; String defaultCustomLayout; + String forcedVideoCodec; + Boolean allowTranscoding; try { mediaModeString = (String) params.get("mediaMode"); recordingModeString = (String) params.get("recordingMode"); @@ -109,6 +112,8 @@ public class SessionRestController { defaultRecordingLayoutString = (String) params.get("defaultRecordingLayout"); defaultCustomLayout = (String) params.get("defaultCustomLayout"); customSessionId = (String) params.get("customSessionId"); + forcedVideoCodec = (String) params.get("forcedVideoCodec"); + allowTranscoding = (Boolean) params.get("allowTranscoding"); } catch (ClassCastException e) { return this.generateErrorResponse("Type error in some parameter", "/api/sessions", HttpStatus.BAD_REQUEST); @@ -150,6 +155,16 @@ public class SessionRestController { builder = builder.customSessionId(customSessionId); } builder = builder.defaultCustomLayout((defaultCustomLayout != null) ? defaultCustomLayout : ""); + if (forcedVideoCodec != null) { + builder = builder.forcedVideoCodec(VideoCodec.valueOf(forcedVideoCodec)); + } else { + builder = builder.forcedVideoCodec(VideoCodec.VP8); + } + if (allowTranscoding != null) { + builder = builder.allowTranscoding(allowTranscoding); + } else { + builder = builder.allowTranscoding(false); + } } catch (IllegalArgumentException e) { return this.generateErrorResponse("RecordingMode " + params.get("recordingMode") + " | " diff --git a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java index 4e742517..f9cc6f00 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rpc/RpcHandler.java @@ -26,6 +26,11 @@ import java.util.concurrent.ConcurrentMap; import javax.servlet.http.HttpSession; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; + import org.apache.commons.lang3.RandomStringUtils; import org.kurento.jsonrpc.DefaultJsonRpcHandler; import org.kurento.jsonrpc.Session; @@ -37,11 +42,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; - import io.openvidu.client.OpenViduException; import io.openvidu.client.OpenViduException.Code; import io.openvidu.client.internal.ProtocolElements; diff --git a/openvidu-server/src/main/java/io/openvidu/server/utils/SDPMunging.java b/openvidu-server/src/main/java/io/openvidu/server/utils/SDPMunging.java new file mode 100644 index 00000000..8537c39e --- /dev/null +++ b/openvidu-server/src/main/java/io/openvidu/server/utils/SDPMunging.java @@ -0,0 +1,133 @@ +package io.openvidu.server.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.openvidu.client.OpenViduException; +import io.openvidu.client.OpenViduException.Code; +import io.openvidu.java.client.VideoCodec; + +public class SDPMunging { + + private static final Logger log = LoggerFactory.getLogger(SDPMunging.class); + + /** + * `codec` is a uppercase SDP-style codec name: "VP8", "H264". + * + * This looks for all video m-sections (lines starting with "m=video"), + * then searches all of its related PayloadTypes trying to find those which + * correspond to the preferred codec. If any is found, they are moved to the + * front of the PayloadTypes list in the m= line, without removing the other + * codecs that might be present. + * + * If our preferred codec is not found, the m= line is left without changes. + * + * This works based on the basis that RFC 3264 "Offer/Answer Model SDP" section + * 6.1 "Unicast Streams" allows the answerer to list media formats in a + * different order of preference from what it got in the offer: + * + * > Although the answerer MAY list the formats in their desired order of + * > preference, it is RECOMMENDED that unless there is a specific reason, + * > the answerer list formats in the same relative order they were + * > present in the offer. + * + * Here we have a specific reason, thus we use this allowance to change the + * ordering of formats. Browsers (tested with Chrome 84) honor this change and + * use the first codec provided in the answer, so this operation actually works. + */ + public String setCodecPreference(VideoCodec codec, String sdp) throws OpenViduException { + String codecStr = codec.name(); + log.info("[setCodecPreference] codec: {}", codecStr); + + List codecPts = new ArrayList(); + String[] lines = sdp.split("\\R+"); + Pattern ptRegex = Pattern.compile(String.format("a=rtpmap:(\\d+) %s/90000", codecStr)); + + for (int sl = 0; sl < lines.length; sl++) { + String sdpLine = lines[sl]; + + if (!sdpLine.startsWith("m=video")) { + continue; + } + + // m-section found. Prepare an array to store PayloadTypes. + codecPts.clear(); + + // Search the m-section to find our codec's PayloadType, if any. + for (int ml = sl + 1; ml < lines.length; ml++) { + String mediaLine = lines[ml]; + + // Abort if we reach the next m-section. + if (mediaLine.startsWith("m=")) { + break; + } + + Matcher ptMatch = ptRegex.matcher(mediaLine); + if (ptMatch.find()) { + // PayloadType found. + String pt = ptMatch.group(1); + codecPts.add(pt); + + // Search the m-section to find the APT subtype, if any. + Pattern aptRegex = Pattern.compile(String.format("a=fmtp:(\\d+) apt=%s", pt)); + + for (int al = sl + 1; al < lines.length; al++) { + String aptLine = lines[al]; + + // Abort if we reach the next m-section. + if (aptLine.startsWith("m=")) { + break; + } + + Matcher aptMatch = aptRegex.matcher(aptLine); + if (aptMatch.find()) { + // APT found. + String apt = aptMatch.group(1); + codecPts.add(apt); + } + } + } + } + + if (codecPts.isEmpty()) { + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, "The specified forced codec " + codecStr + " is not present in the SDP"); + } + + // Build a new m= line where any PayloadTypes found have been moved + // to the front of the PT list. + StringBuilder newLine = new StringBuilder(sdpLine.length()); + List lineParts = new ArrayList(Arrays.asList(sdpLine.split(" "))); + + if (lineParts.size() < 4) { + log.error("[setCodecPreference] BUG in m= line: Expects at least 4 fields: '{}'", sdpLine); + continue; + } + + // Add "m=video", Port, and Protocol. + for (int i = 0; i < 3; i++) { + newLine.append(lineParts.remove(0) + " "); + } + + // Add the PayloadTypes that correspond to our preferred codec. + for (String pt : codecPts) { + lineParts.remove(pt); + newLine.append(pt + " "); + } + + // Add the rest of PayloadTypes. + newLine.append(String.join(" ", lineParts)); + + // Replace the original m= line with the one we just built. + lines[sl] = newLine.toString(); + } + + return String.join("\r\n", lines); + } + +} \ No newline at end of file diff --git a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.css b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.css index 5c4c9594..782565c8 100644 --- a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.css +++ b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.css @@ -36,4 +36,8 @@ mat-radio-button:first-child { #role-div { padding-top: 6px; padding-bottom: 15px; +} + +#allow-transcoding-div { + margin-bottom: 10px; } \ No newline at end of file diff --git a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html index de5608d5..bf816c9b 100644 --- a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html +++ b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html @@ -38,6 +38,18 @@ +
+ Allow Transcoding +
+ + + + {{ enumerator }} + + + diff --git a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.ts b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.ts index b7427ed2..f06f3201 100644 --- a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.ts +++ b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.ts @@ -1,7 +1,7 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material'; -import { SessionProperties, MediaMode, Recording, RecordingMode, RecordingLayout, TokenOptions } from 'openvidu-node-client'; +import { SessionProperties, MediaMode, Recording, RecordingMode, RecordingLayout, TokenOptions, VideoCodec } from 'openvidu-node-client'; @Component({ selector: 'app-session-properties-dialog', @@ -23,6 +23,7 @@ export class SessionPropertiesDialogComponent { recordingMode = RecordingMode; defaultOutputMode = Recording.OutputMode; defaultRecordingLayout = RecordingLayout; + forceVideoCodec = VideoCodec; constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data) { diff --git a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts index eda242a8..fc324f9f 100644 --- a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts +++ b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts @@ -18,7 +18,8 @@ import { TokenOptions, OpenViduRole, RecordingProperties, - Recording + Recording, + VideoCodec } from 'openvidu-node-client'; import { MatDialog, MAT_CHECKBOX_CLICK_ACTION } from '@angular/material'; import { ExtensionDialogComponent } from '../dialogs/extension-dialog/extension-dialog.component'; @@ -92,7 +93,9 @@ export class OpenviduInstanceComponent implements OnInit, OnChanges, OnDestroy { defaultOutputMode: Recording.OutputMode.COMPOSED, defaultRecordingLayout: RecordingLayout.BEST_FIT, defaultCustomLayout: '', - customSessionId: '' + customSessionId: '', + forcedVideoCodec: VideoCodec.VP8, + allowTranscoding: false }; publisherProperties: PublisherProperties = { From 0f958baf39cc0ba5023bb90e1176354d211e1a55 Mon Sep 17 00:00:00 2001 From: cruizba Date: Fri, 4 Sep 2020 17:33:12 +0200 Subject: [PATCH 7/7] Fixed forceVideoCodec and allowTranscoding properties. Add debug logs for SDP --- .../java/client/SessionProperties.java | 29 ++----- .../io/openvidu/java/client/VideoCodec.java | 2 +- openvidu-node-client/src/SessionProperties.ts | 28 +++---- openvidu-node-client/src/VideoCodec.ts | 3 +- .../deployments/ce/docker-compose/.env | 7 ++ .../docker-compose/openvidu-server-pro/.env | 7 ++ .../server/config/OpenviduConfig.java | 52 ++++++++---- .../kurento/core/KurentoSessionManager.java | 84 +++++++++++-------- .../server/rest/SessionRestController.java | 4 +- .../src/main/resources/application.properties | 3 + .../session-properties-dialog.component.html | 2 +- .../app/components/video/video.component.html | 3 + .../app/components/video/video.component.ts | 7 ++ 13 files changed, 137 insertions(+), 94 deletions(-) diff --git a/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java b/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java index 59842409..0f5ae90f 100644 --- a/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java +++ b/openvidu-java-client/src/main/java/io/openvidu/java/client/SessionProperties.java @@ -144,11 +144,12 @@ public class SessionProperties { } /** - * * Call this method to define which video codec do you want to be forcibly used for this session. - * This allows browsers to use the same codec avoiding transcoding in the media server. - * To force this video codec you need to set {@link #allowTranscoding(boolean)} to false. + * This allows browsers/clients to use the same codec avoiding transcoding in the media server. + * If the browser/client is not compatible with the specified codec and {@link #allowTranscoding(boolean)} + * is false and exception will occur. * + * If forcedVideoCodec is set to NONE, no codec will be forced. */ public SessionProperties.Builder forcedVideoCodec(VideoCodec forcedVideoCodec) { this.forcedVideoCodec = forcedVideoCodec; @@ -156,15 +157,8 @@ public class SessionProperties { } /** - * - * Call this method to define if you want to allowTranscoding or not. If you define it as - * as false, the default video codec VP8 will be used for all browsers, and the media - * server will not do any transcoding. If you define it as true, transcoding can be - * executed by the media server when necessary. - * - * If you want to set a different video codec, you can configure it - * by calling {@link #forcedVideoCodec(VideoCodec)} to your preferred one. - * + * Call this method to define if you want to allow transcoding in the media server or not + * when {@link #forcedVideoCodec(VideoCodec)} is not compatible with the browser/client. */ public SessionProperties.Builder allowTranscoding(boolean allowTranscoding) { this.allowTranscoding = allowTranscoding; @@ -270,20 +264,15 @@ public class SessionProperties { } /** - * - * Defines which video codec is being forced to be used when - * {@link io.openvidu.java.client.SessionProperties.Builder#allowTranscoding(boolean)} - * has been set to false + * Defines which video codec is being forced to be used in the browser/client */ public VideoCodec forcedVideoCodec() { return this.forcedVideoCodec; } /** - * - * Defines if transcoding is allowed or not. If this method returns false, a video codec - * will be forcibly used for all browsers (See - * {@link io.openvidu.java.client.SessionProperties.Builder#forcedVideoCodec(VideoCodec)}). + * Defines if transcoding is allowed or not when {@link #forcedVideoCodec} + * is not a compatible codec with the browser/client. */ public boolean isTranscodingAllowed() { return this.allowTranscoding; diff --git a/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java b/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java index 9f6e002f..3f088fa1 100644 --- a/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java +++ b/openvidu-java-client/src/main/java/io/openvidu/java/client/VideoCodec.java @@ -21,5 +21,5 @@ package io.openvidu.java.client; * See {@link io.openvidu.java.client.SessionProperties.Builder#forcedVideoCodec(VideoCodec)} */ public enum VideoCodec { - VP8, VP9, H264 + VP8, VP9, H264, NONE } \ No newline at end of file diff --git a/openvidu-node-client/src/SessionProperties.ts b/openvidu-node-client/src/SessionProperties.ts index df45ac41..91e182c8 100644 --- a/openvidu-node-client/src/SessionProperties.ts +++ b/openvidu-node-client/src/SessionProperties.ts @@ -19,6 +19,7 @@ import { MediaMode } from './MediaMode'; import { Recording } from './Recording'; import { RecordingLayout } from './RecordingLayout'; import { RecordingMode } from './RecordingMode'; +import { VideoCodec } from './VideoCodec'; /** * See [[OpenVidu.createSession]] @@ -65,23 +66,20 @@ export interface SessionProperties { */ customSessionId?: string; - /** - * Call this method to define which video codec do you want to be forcibly used for this session. - * This allows browsers to use the same codec avoiding transcoding in the media server. - * To force this video codec you need to set [[allowTranscoding]] to false. - */ - forcedVideoCodec?: string; + /** + * It defines which video codec do you want to be forcibly used for this session. + * This allows browsers/clients to use the same codec avoiding transcoding in the media server. + * If the browser/client is not compatible with the specified codec and [[allowTranscoding]] + * is false and exception will occur. + * + * If forcedVideoCodec is set to NONE, no codec will be forced. + */ + forcedVideoCodec?: VideoCodec; /** - * Call this method to define if you want to allowTranscoding or not. If you define it as - * as false, the default video codec VP8 will be used for all browsers, and the media - * server will not do any transcoding. If you define it as true, transcoding can be - * executed by the media server when necessary. - * - * If you want to set a different video codec, you can configure it - * by calling [[forcedVideoCodec]] to your preferred one. - * - */ + * It defines if you want to allow transcoding in the media server or not + * when [[forcedVideoCodec]] is not compatible with the browser/client. + */ allowTranscoding?: boolean; } diff --git a/openvidu-node-client/src/VideoCodec.ts b/openvidu-node-client/src/VideoCodec.ts index e1077f77..464bef34 100644 --- a/openvidu-node-client/src/VideoCodec.ts +++ b/openvidu-node-client/src/VideoCodec.ts @@ -5,6 +5,7 @@ export enum VideoCodec { VP8 = 'VP8', VP9 = 'VP9', - H264 = 'H264' + H264 = 'H264', + NONE = 'NONE' } \ No newline at end of file diff --git a/openvidu-server/deployments/ce/docker-compose/.env b/openvidu-server/deployments/ce/docker-compose/.env index 29ac0cb5..fe1e7693 100644 --- a/openvidu-server/deployments/ce/docker-compose/.env +++ b/openvidu-server/deployments/ce/docker-compose/.env @@ -137,6 +137,13 @@ OPENVIDU_SESSIONS_GARBAGE_INTERVAL=900 # (property 'OPENVIDU_SESSIONS_GARBAGE_INTERVAL' to 0) this property is ignored OPENVIDU_SESSIONS_GARBAGE_THRESHOLD=3600 +# All sessions of OpenVidu will try to force this codec. If OPENVIDU_ALLOW_TRANSCODING=true +# when a codec can not be forced, transcoding will be allowed +# OPENVIDU_FORCED_CODEC=VP8 + +# Allow transcoding if codec specified in OPENVIDU_FORCED_CODEC can not be applied +# OPENVIDU_ALLOW_TRANSCODING=false + # Call Detail Record enabled # Whether to enable Call Detail Record or not # Values: true | false diff --git a/openvidu-server/deployments/pro/docker-compose/openvidu-server-pro/.env b/openvidu-server/deployments/pro/docker-compose/openvidu-server-pro/.env index 056cf286..458697fa 100644 --- a/openvidu-server/deployments/pro/docker-compose/openvidu-server-pro/.env +++ b/openvidu-server/deployments/pro/docker-compose/openvidu-server-pro/.env @@ -178,6 +178,13 @@ OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH=1000 # 0 means unconstrained OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 +# All sessions of OpenVidu will try to force this codec. If OPENVIDU_ALLOW_TRANSCODING=true +# when a codec can not be forced, transcoding will be allowed +# OPENVIDU_FORCED_CODEC=VP8 + +# Allow transcoding if codec specified in OPENVIDU_FORCED_CODEC can not be applied +# OPENVIDU_ALLOW_TRANSCODING=false + # true to enable OpenVidu Webhook service. false' otherwise # Values: true | false OPENVIDU_WEBHOOK=false diff --git a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java index 1beb6a41..e2b27fe0 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java +++ b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java @@ -37,6 +37,10 @@ import java.util.Map; import javax.annotation.PostConstruct; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonSyntaxException; + import org.apache.commons.io.FilenameUtils; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -48,11 +52,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonSyntaxException; - import io.openvidu.java.client.OpenViduRole; +import io.openvidu.java.client.VideoCodec; import io.openvidu.server.OpenViduServer; import io.openvidu.server.cdr.CDREventName; import io.openvidu.server.config.Dotenv.DotenvFormatException; @@ -176,6 +177,10 @@ public class OpenviduConfig { protected int openviduSessionsGarbageThreshold; + private VideoCodec openviduForcedCodec; + + private boolean openviduAllowTranscoding; + private String dotenvPath; // Derived properties @@ -190,6 +195,14 @@ public class OpenviduConfig { return this.coturnRedisDbname; } + public boolean isOpenviduAllowingTranscoding() { + return openviduAllowTranscoding; + } + + public VideoCodec getOpenviduForcedCodec() { + return openviduForcedCodec; + } + public String getCoturnDatabasePassword() { return this.coturnRedisPassword; } @@ -335,20 +348,20 @@ public class OpenviduConfig { public OpenViduRole[] getRolesFromRecordingNotification() { OpenViduRole[] roles; switch (this.openviduRecordingNotification) { - case none: - roles = new OpenViduRole[0]; - break; - case moderator: - roles = new OpenViduRole[] { OpenViduRole.MODERATOR }; - break; - case publisher_moderator: - roles = new OpenViduRole[] { OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; - break; - case all: - roles = new OpenViduRole[] { OpenViduRole.SUBSCRIBER, OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; - break; - default: - roles = new OpenViduRole[] { OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; + case none: + roles = new OpenViduRole[0]; + break; + case moderator: + roles = new OpenViduRole[] { OpenViduRole.MODERATOR }; + break; + case publisher_moderator: + roles = new OpenViduRole[] { OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; + break; + case all: + roles = new OpenViduRole[] { OpenViduRole.SUBSCRIBER, OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; + break; + default: + roles = new OpenViduRole[] { OpenViduRole.PUBLISHER, OpenViduRole.MODERATOR }; } return roles; } @@ -500,6 +513,9 @@ public class OpenviduConfig { openviduSessionsGarbageInterval = asNonNegativeInteger("OPENVIDU_SESSIONS_GARBAGE_INTERVAL"); openviduSessionsGarbageThreshold = asNonNegativeInteger("OPENVIDU_SESSIONS_GARBAGE_THRESHOLD"); + openviduForcedCodec = asEnumValue("OPENVIDU_FORCED_CODEC", VideoCodec.class); + openviduAllowTranscoding = asBoolean("OPENVIDU_ALLOW_TRANSCODING"); + kmsUrisList = checkKmsUris(); checkCoturnIp(); diff --git a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java index fb4399b7..27a7fd32 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java +++ b/openvidu-server/src/main/java/io/openvidu/server/kurento/core/KurentoSessionManager.java @@ -371,20 +371,26 @@ public class KurentoSessionManager extends SessionManager { KurentoMediaOptions kurentoOptions = (KurentoMediaOptions) mediaOptions; KurentoParticipant kParticipant = (KurentoParticipant) participant; KurentoSession kSession = kParticipant.getSession(); - - // Modify sdp if transcoding is not allowed - if(!kSession.getSessionProperties().isTranscodingAllowed()) { - VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + boolean isTranscodingAllowed = kSession.getSessionProperties().isTranscodingAllowed(); + VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + + // Modify sdp if forced codec is defined + if (forcedVideoCodec != VideoCodec.NONE) { String sdpOffer = kurentoOptions.sdpOffer; - try { - kurentoOptions.sdpOffer = modifySdpToForceCodec(forcedVideoCodec, sdpOffer); + log.debug("PARTICIPANT '{}' in Session '{}' SDP Offer before munging: \n {}", + participant.getParticipantPublicId(), kSession.getSessionId(), kurentoOptions.sdpOffer); + kurentoOptions.sdpOffer = this.sdpMunging.setCodecPreference(forcedVideoCodec, sdpOffer); } catch (OpenViduException e) { - String errorMessage = "Error forcing codec: ''" + forcedVideoCodec + "', for publisher on Session: '" + kSession.getSessionId() - + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpOffer; - throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + String errorMessage = "Error forcing codec: '" + forcedVideoCodec + "', for PARTICIPANT" + + participant.getParticipantPublicId() + "' publishing in Session: '" + + kSession.getSessionId() + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpAnswer; + if(!isTranscodingAllowed) { + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + log.info("Codec: '" + forcedVideoCodec + "' is not supported for PARTICIPANT: '" + participant.getParticipantPublicId() + + " publishing in Session: '" + kSession.getSessionId() + "'. Transcoding will be allowed"); } - } log.debug( @@ -418,7 +424,7 @@ public class KurentoSessionManager extends SessionManager { } sdpAnswer = kParticipant.publishToRoom(kurentoOptions.sdpOffer, kurentoOptions.doLoopback, false); - + if (sdpAnswer == null) { OpenViduException e = new OpenViduException(Code.MEDIA_SDP_ERROR_CODE, "Error generating SDP response for publishing user " + participant.getParticipantPublicId()); @@ -582,16 +588,25 @@ public class KurentoSessionManager extends SessionManager { KurentoParticipant kParticipant = (KurentoParticipant) participant; session = ((KurentoParticipant) participant).getSession(); Participant senderParticipant = session.getParticipantByPublicId(senderName); + boolean isTranscodingAllowed = session.getSessionProperties().isTranscodingAllowed(); + VideoCodec forcedVideoCodec = session.getSessionProperties().forcedVideoCodec(); - // Modify sdp if transcoding is not allowed - if (!session.getSessionProperties().isTranscodingAllowed()) { - VideoCodec forcedVideoCodec = session.getSessionProperties().forcedVideoCodec(); + // Modify sdp if forced codec is defined + if (forcedVideoCodec != VideoCodec.NONE) { try { - sdpAnswer = this.modifySdpToForceCodec(forcedVideoCodec, sdpAnswer); + log.debug("PARTICIPANT '{}' in Session '{}' SDP Answer before munging: \n {}", + participant.getParticipantPublicId(), session.getSessionId(), sdpAnswer); + sdpAnswer = this.sdpMunging.setCodecPreference(forcedVideoCodec, sdpAnswer); } catch (OpenViduException e) { - String errorMessage = "Error forcing codec: ''" + forcedVideoCodec + "', for subscriber on Session: '" + String errorMessage = "Error forcing codec: '" + forcedVideoCodec + "', for PARTICIPANT: '" + + participant.getParticipantPublicId() + "' subscribing in Session: '" + session.getSessionId() + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpAnswer; - throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + + if(!isTranscodingAllowed) { + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + log.info("Codec: '" + forcedVideoCodec + "' is not supported for PARTICIPANT: '" + participant.getParticipantPublicId() + + " subscribing in Session: '" + session.getSessionId() + "'. Transcoding will be allowed"); } } @@ -1126,19 +1141,26 @@ public class KurentoSessionManager extends SessionManager { KurentoParticipant kParticipant = (KurentoParticipant) participant; KurentoSession kSession = kParticipant.getSession(); boolean isPublisher = streamId.equals(participant.getPublisherStreamId()); + boolean isTranscodingAllowed = kSession.getSessionProperties().isTranscodingAllowed(); + VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); - // Modify sdp if transcoding is not allowed - if (!kSession.getSessionProperties().isTranscodingAllowed()) { - VideoCodec forcedVideoCodec = kSession.getSessionProperties().forcedVideoCodec(); + // Modify sdp if forced codec is defined + if (forcedVideoCodec != VideoCodec.NONE) { try { - sdpString = modifySdpToForceCodec(forcedVideoCodec, sdpString); + log.debug("PARTICIPANT '{}' in Session '{}' reconnecting SDP before munging: \n {}", + participant.getParticipantPublicId(), kSession.getSessionId(), sdpString); + sdpString = sdpMunging.setCodecPreference(forcedVideoCodec, sdpString); } catch (OpenViduException e) { - String errorMessage = "Error on reconnecting and forcing codec: ''" + forcedVideoCodec + "', for " - + (isPublisher ? "publisher" : "subscriber") + " on Session: '" + kSession.getSessionId() - + "'\nException: " + e.getMessage() + "\nSDP:\n" + sdpString; - throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + String errorMessage = "Error in reconnect and forcing codec: '" + forcedVideoCodec + "', for PARTICIPANT: '" + + participant.getParticipantPublicId() + "' " + (isPublisher ? "publishing" : "subscribing") + + " in Session: '" + kSession.getSessionId() + "'\nException: " + + e.getMessage() + "\nSDP:\n" + sdpString; + if(!isTranscodingAllowed) { + throw new OpenViduException(Code.FORCED_CODEC_NOT_FOUND_IN_SDPOFFER, errorMessage); + } + log.info("Codec: '" + forcedVideoCodec + "' is not supported for PARTICIPANT: '" + participant.getParticipantPublicId() + + "' " + (isPublisher ? "publishing" : "subscribing") + " in Session: '" + kSession.getSessionId() + "'. Transcoding will be allowed"); } - } if (isPublisher) { @@ -1243,14 +1265,4 @@ public class KurentoSessionManager extends SessionManager { filter.removeEventListener(pub.removeListener(eventType)); } } - - - private String modifySdpToForceCodec(VideoCodec codec, String sdpOffer) { - // Modify sdpOffer if transcoding is not allowed - String modSdpOffer = this.sdpMunging.setCodecPreference(codec, sdpOffer); - if (modSdpOffer != null) { - sdpOffer = modSdpOffer; - } - return sdpOffer; - } } diff --git a/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java b/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java index e773c314..76b1470b 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rest/SessionRestController.java @@ -158,12 +158,12 @@ public class SessionRestController { if (forcedVideoCodec != null) { builder = builder.forcedVideoCodec(VideoCodec.valueOf(forcedVideoCodec)); } else { - builder = builder.forcedVideoCodec(VideoCodec.VP8); + builder = builder.forcedVideoCodec(openviduConfig.getOpenviduForcedCodec()); } if (allowTranscoding != null) { builder = builder.allowTranscoding(allowTranscoding); } else { - builder = builder.allowTranscoding(false); + builder = builder.allowTranscoding(openviduConfig.isOpenviduAllowingTranscoding()); } } catch (IllegalArgumentException e) { diff --git a/openvidu-server/src/main/resources/application.properties b/openvidu-server/src/main/resources/application.properties index 48b9859a..2ae45ad0 100644 --- a/openvidu-server/src/main/resources/application.properties +++ b/openvidu-server/src/main/resources/application.properties @@ -42,6 +42,9 @@ OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 OPENVIDU_SESSIONS_GARBAGE_INTERVAL=900 OPENVIDU_SESSIONS_GARBAGE_THRESHOLD=3600 +OPENVIDU_FORCED_CODEC=VP8 +OPENVIDU_ALLOW_TRANSCODING=false + COTURN_REDIS_IP=127.0.0.1 COTURN_REDIS_DBNAME=0 COTURN_REDIS_PASSWORD=turn diff --git a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html index bf816c9b..e9eb8de6 100644 --- a/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html +++ b/openvidu-testapp/src/app/components/dialogs/session-properties-dialog/session-properties-dialog.component.html @@ -42,7 +42,7 @@ Allow Transcoding - + diff --git a/openvidu-testapp/src/app/components/video/video.component.html b/openvidu-testapp/src/app/components/video/video.component.html index 5e52c0e4..7ffe5117 100644 --- a/openvidu-testapp/src/app/components/video/video.component.html +++ b/openvidu-testapp/src/app/components/video/video.component.html @@ -5,6 +5,9 @@ +