From 8e5f5d4cf4c138cab994edf93d7b08849f2439e1 Mon Sep 17 00:00:00 2001 From: Juan Navarro Date: Wed, 5 Jan 2022 15:12:51 +0100 Subject: [PATCH] openvidu-server, browser: Add Simulcast Publisher config (#680) Simulcast is a per-Publisher configuration that allows to enable Simulcast senders on the client's PeerConnection of each sender. Simulcast is a WebRTC feature that sends multiple simultaneous streams with different video qualities, in order to let the media server decide which quality is best for which Subscriber on the receiving side. Enabled by default. --- openvidu-browser/src/OpenVidu/OpenVidu.ts | 5 +++++ openvidu-browser/src/OpenVidu/Session.ts | 1 + openvidu-browser/src/OpenVidu/Stream.ts | 8 +++++++- .../Interfaces/Private/LocalConnectionOptions.ts | 3 ++- .../Public/OpenViduAdvancedConfiguration.ts | 9 +-------- .../Interfaces/Public/PublisherProperties.ts | 12 +++++++++++- .../openvidu/client/internal/ProtocolElements.java | 1 + .../deployments/enterprise/master-node/.env | 10 ++++++++++ .../io/openvidu/server/config/OpenviduConfig.java | 9 +++++++++ .../openvidu/server/core/SessionEventsHandler.java | 13 +++++++++++++ .../openvidu/server/rest/ConfigRestController.java | 1 + .../additional-spring-configuration-metadata.json | 6 ++++++ .../src/main/resources/application.properties | 1 + .../publisher-properties-dialog.component.html | 3 ++- .../openvidu-instance.component.ts | 7 +++---- 15 files changed, 73 insertions(+), 16 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/OpenVidu.ts b/openvidu-browser/src/OpenVidu/OpenVidu.ts index e2de4ff6..34bdeea1 100644 --- a/openvidu-browser/src/OpenVidu/OpenVidu.ts +++ b/openvidu-browser/src/OpenVidu/OpenVidu.ts @@ -109,6 +109,10 @@ export class OpenVidu { * @hidden */ mediaServer: string; + /** + * @hidden + */ + videoSimulcast: boolean; /** * @hidden */ @@ -257,6 +261,7 @@ export class OpenVidu { publishVideo: (typeof properties.publishVideo !== 'undefined') ? properties.publishVideo : true, resolution: (typeof MediaStreamTrack !== 'undefined' && properties.videoSource instanceof MediaStreamTrack) ? undefined : ((typeof properties.resolution !== 'undefined') ? properties.resolution : '640x480'), videoSource: (typeof properties.videoSource !== 'undefined') ? properties.videoSource : undefined, + videoSimulcast: properties.videoSimulcast, filter: properties.filter }; } else { diff --git a/openvidu-browser/src/OpenVidu/Session.ts b/openvidu-browser/src/OpenVidu/Session.ts index ad243837..b25c7e10 100644 --- a/openvidu-browser/src/OpenVidu/Session.ts +++ b/openvidu-browser/src/OpenVidu/Session.ts @@ -1540,6 +1540,7 @@ export class Session extends EventDispatcher { this.openvidu.role = opts.role; this.openvidu.finalUserId = opts.finalUserId; this.openvidu.mediaServer = opts.mediaServer; + this.openvidu.videoSimulcast = opts.videoSimulcast; this.capabilities = { subscribe: true, publish: this.openvidu.role !== 'SUBSCRIBER', diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index 16d8c9b2..ae5cf795 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -938,7 +938,8 @@ export class Stream { audio: this.hasAudio, video: this.hasVideo, }, - simulcast: this.session.openvidu.advancedConfiguration.enableSimulcastExperimental || false, + simulcast: + this.outboundStreamOpts.publisherProperties.videoSimulcast ?? this.session.openvidu.videoSimulcast, onIceCandidate: this.connection.sendIceCandidate.bind(this.connection), onIceConnectionStateException: (exceptionName: ExceptionEventName, message: string, data?: any) => { this.session.emitEvent('exception', [new ExceptionEvent(this.session, exceptionName, this, message, data)]) }, iceServers: this.getIceServersConf(), @@ -946,6 +947,11 @@ export class Stream { mediaServer: this.session.openvidu.mediaServer }; + if (this.session.openvidu.mediaServer !== 'mediasoup') { + // Simulcast is only supported by mediasoup + config.simulcast = false; + } + if (reconnect) { this.disposeWebRtcPeer(); } diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts index bc8d2ab3..8520aad1 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts @@ -33,5 +33,6 @@ export interface LocalConnectionOptions { turnCredential: string; version: string; mediaServer: string; + videoSimulcast: boolean; life: number; -} \ No newline at end of file +} diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts index 2bc8f046..7327b3c2 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts @@ -71,11 +71,4 @@ export interface OpenViduAdvancedConfiguration { */ noStreamPlayingEventExceptionTimeout?: number; - /** - * Whether to enable simulcast for Publishers or not. - * - * Default to `false`. - */ - enableSimulcastExperimental?: boolean; - -} \ No newline at end of file +} diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts index 2ef5df08..dbe0c715 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts @@ -82,6 +82,16 @@ export interface PublisherProperties { */ videoSource?: string | MediaStreamTrack | boolean; + /** + * Send Simulcast video. + * Publishers will encode duplicate video streams with different qualities, + * so the media server is able to select the most appropriate quality stream + * for each Subscriber. + * This setting is honored only if OpenVidu Server was configured to use the + * mediasoup media server. Otherwise, Simulcast will be disabled. + */ + videoSimulcast?: boolean; + /** * **WARNING**: experimental option. This property may change in the near future * @@ -89,4 +99,4 @@ export interface PublisherProperties { */ filter?: Filter; -} \ No newline at end of file +} diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index 4e917016..85a327d5 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -160,6 +160,7 @@ public class ProtocolElements { public static final String PARTICIPANTJOINED_SESSION_PARAM = "session"; public static final String PARTICIPANTJOINED_VERSION_PARAM = "version"; public static final String PARTICIPANTJOINED_MEDIASERVER_PARAM = "mediaServer"; + public static final String PARTICIPANTJOINED_SIMULCAST_PARAM = "videoSimulcast"; public static final String PARTICIPANTJOINED_RECORD_PARAM = "record"; public static final String PARTICIPANTJOINED_ROLE_PARAM = "role"; public static final String PARTICIPANTJOINED_COTURNIP_PARAM = "coturnIp"; diff --git a/openvidu-server/deployments/enterprise/master-node/.env b/openvidu-server/deployments/enterprise/master-node/.env index ace47e9f..1db77389 100644 --- a/openvidu-server/deployments/enterprise/master-node/.env +++ b/openvidu-server/deployments/enterprise/master-node/.env @@ -247,6 +247,16 @@ OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH=1000 # 0 means unconstrained OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 +# Send Simulcast video. +# Publishers will encode duplicate video streams with different qualities, +# so the media server is able to select the most appropriate quality stream +# for each Subscriber. +# This setting is honored only if OpenVidu Server was configured to use the +# mediasoup media server. Otherwise, Simulcast will be disabled. +# Values: true | false +# Default: true +#OPENVIDU_STREAMS_VIDEO_SIMULCAST=true + # All sessions of OpenVidu will try to force this codec. If OPENVIDU_STREAMS_ALLOW_TRANSCODING=true # when a codec can not be forced, transcoding will be allowed # Values: VP8, VP9, H264, NONE diff --git a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java index dede7ea0..47fc14b1 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java +++ b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java @@ -217,8 +217,13 @@ public class OpenviduConfig { private boolean isTurnadminAvailable = false; // Media Server properties + private MediaServer mediaServerInfo = MediaServer.kurento; + // Media properties + + private boolean streamsVideoSimulcast = false; + // Plain config properties getters public String getCoturnDatabaseDbname() { @@ -281,6 +286,10 @@ public class OpenviduConfig { this.mediaServerInfo = mediaServerInfo; } + public boolean isStreamsVideoSimulcast() { + return this.streamsVideoSimulcast; + } + public String getOpenViduRecordingPath() { return this.openviduRecordingPath; } diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java index 3c54b939..2ec4fd68 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java @@ -165,6 +165,19 @@ public class SessionEventsHandler { result.addProperty(ProtocolElements.PARTICIPANTJOINED_MEDIASERVER_PARAM, this.openviduConfig.getMediaServer().name()); + switch (this.openviduConfig.getMediaServer()) { + case mediasoup: + // mediasoup supports simulcast + result.addProperty(ProtocolElements.PARTICIPANTJOINED_SIMULCAST_PARAM, + this.openviduConfig.isStreamsVideoSimulcast()); + break; + case kurento: + default: + // Kurento does not support simulcast + result.addProperty(ProtocolElements.PARTICIPANTJOINED_SIMULCAST_PARAM, false); + break; + } + if (participant.getToken() != null) { result.addProperty(ProtocolElements.PARTICIPANTJOINED_RECORD_PARAM, participant.getToken().record()); if (participant.getToken().getRole() != null) { diff --git a/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java b/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java index 9c1d1e2f..3f3edc39 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java @@ -114,6 +114,7 @@ public class ConfigRestController { json.addProperty("OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH", openviduConfig.getVideoMinRecvBandwidth()); json.addProperty("OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH", openviduConfig.getVideoMaxSendBandwidth()); json.addProperty("OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH", openviduConfig.getVideoMinSendBandwidth()); + json.addProperty("OPENVIDU_STREAMS_VIDEO_SIMULCAST", openviduConfig.isStreamsVideoSimulcast()); json.addProperty("OPENVIDU_STREAMS_FORCED_VIDEO_CODEC", openviduConfig.getOpenviduForcedCodec().name()); json.addProperty("OPENVIDU_STREAMS_ALLOW_TRANSCODING", openviduConfig.isOpenviduAllowingTranscoding()); json.addProperty("OPENVIDU_SESSIONS_GARBAGE_INTERVAL", openviduConfig.getSessionGarbageInterval()); diff --git a/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 6ba031d8..e34d6a5e 100644 --- a/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -147,6 +147,12 @@ "description": "Minimum video bandwidth sent from OpenVidu Server to clients, in kbps. 0 means unconstrained", "defaultValue": 300 }, + { + "name": "OPENVIDU_STREAMS_VIDEO_SIMULCAST", + "type": "java.lang.Boolean", + "description": "Send Simulcast video.", + "defaultValue": true + }, { "name": "OPENVIDU_STREAMS_FORCED_VIDEO_CODEC", "type": "java.lang.String", diff --git a/openvidu-server/src/main/resources/application.properties b/openvidu-server/src/main/resources/application.properties index fe3076e7..6311b88a 100644 --- a/openvidu-server/src/main/resources/application.properties +++ b/openvidu-server/src/main/resources/application.properties @@ -42,6 +42,7 @@ OPENVIDU_STREAMS_VIDEO_MAX_RECV_BANDWIDTH=1000 OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH=300 OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH=1000 OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 +OPENVIDU_STREAMS_VIDEO_SIMULCAST=true OPENVIDU_STREAMS_FORCED_VIDEO_CODEC=VP8 OPENVIDU_STREAMS_ALLOW_TRANSCODING=false diff --git a/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html b/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html index 816c8b6b..2044560c 100644 --- a/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html +++ b/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html @@ -8,6 +8,7 @@ Publish audio Publish video Mirror + Video Simulcast @@ -52,4 +53,4 @@ - \ No newline at end of file + diff --git a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts index dc46397e..0c8d8a1e 100644 --- a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts +++ b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts @@ -121,7 +121,8 @@ export class OpenviduInstanceComponent implements OnInit, OnChanges, OnDestroy { resolution: '640x480', mirror: true, publishAudio: true, - publishVideo: true + publishVideo: true, + videoSimulcast: true }; publisherPropertiesAux: PublisherProperties; @@ -230,9 +231,7 @@ export class OpenviduInstanceComponent implements OnInit, OnChanges, OnDestroy { this.OV = new OpenVidu(); - const advancedConfiguration: OpenViduAdvancedConfiguration = { - enableSimulcastExperimental: false - }; + const advancedConfiguration: OpenViduAdvancedConfiguration = {}; if (this.turnConf === 'freeice') { advancedConfiguration.iceServers = 'freeice'; } else if (this.turnConf === 'manual') {