openvidu-browser: adaptive simulcast config depending on source video

Do not blindly enable simulcast; instead, analyze the properties of the
source track in order to decide upon a good configuration:

* Take into account Google Chrome decision table which dictates how many
simulcast layers will be sent at most:
https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/media/engine/simulcast.cc;l=90-114;drc=d3251968d1b3dbe7e1353a3f15970b47173103e9

* Limit bitrate and resolution on webcam videos, or framerate on
screenshare videos.

openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts

@@ -83,8 +83,8 @@ export interface PublisherProperties {
     videoSource?: string | MediaStreamTrack | boolean;
 
     /**
-     * Send Simulcast video.
+     * Use Simulcast video on WebRTC Publishers.
-     * Publishers will encode duplicate video streams with different qualities,
+     * Senders will encode duplicate video streams with different qualities,
      * so the media server is able to select the most appropriate quality stream
      * for each Subscriber.
      * This setting is honored only if OpenVidu Server was configured to use the

openvidu-browser/src/OpenViduInternal/WebRtcPeer/WebRtcPeer.ts

@@ -31,71 +31,10 @@ const logger: OpenViduLogger = OpenViduLogger.getInstance();
  */
 let platform: PlatformUtils;
 
-/*
- * Table of sender video encodings for simulcast.
- * Note that this is just a polite request, but the browser is free to honor it
- * or just play by its own rules.
- *
- * Chrome imposes some restrictions based on the size of the video, max bitrate,
- * and available bandwidth. Check here for the video size table:
- * https://chromium.googlesource.com/external/webrtc/+/master/media/engine/simulcast.cc#90
- *
- * | Size (px) | Bitrate (kbps) | Max Layers |
- * |----------:|---------------:|-----------:|
- * | 1920x1080 |           5000 |          3 |
- * |  1280x720 |           2500 |          3 |
- * |   960x540 |           1200 |          3 |
- * |   640x360 |            700 |          2 |
- * |   480x270 |            450 |          2 |
- * |   320x180 |            200 |          1 |
- *
- * Firefox will send as many layers as we request, but there are some limits on
- * their bitrate:
- *
- * | Size (px) | Min bitrate (bps) | Start bitrate (bps) | Max bitrate (bps) |       Comments |
- * |----------:|------------------:|--------------------:|------------------:|---------------:|
- * | 1920x1200 |              1500 |                2000 |             10000 |   >HD (3K, 4K) |
- * |  1280x720 |              1200 |                1500 |              5000 |  HD ~1080-1200 |
- * |   800x480 |               200 |                 800 |              2500 |        HD ~720 |
- * |   480x270 |               150 |                 500 |              2000 |           WVGA |
- * |   400x240 |               125 |                 300 |              1300 |            VGA |
- * |   176x144 |               100 |                 150 |               500 |     WQVGA, CIF |
- * |         0 |                40 |                  80 |               250 | QCIF and below |
- *
- * Docs for `RTCRtpEncodingParameters`: https://www.w3.org/TR/webrtc/#dom-rtcrtpencodingparameters
- * Most interesting members are `maxBitrate` and `scaleResolutionDownBy`.
- *
- * `scaleResolutionDownBy` is specified as 4:2:1 which is the same that the default.
- * The WebRTC spec says this (https://www.w3.org/TR/webrtc/#dom-rtcpeerconnection-addtransceiver):
- *     > If the scaleResolutionDownBy attributes of sendEncodings are still undefined, initialize
- *     > each encoding's scaleResolutionDownBy to 2^(length of sendEncodings - encoding index
- *     > - 1). This results in smaller-to-larger resolutions where the last encoding has no scaling
- *     > applied to it, e.g. 4:2:1 if the length is 3.
- * However, Firefox doesn't seem to implement this default yet. Mediasoup never gets to select
- * an output layer.
- *
- * `maxBitrate` is left unspecified, to let the client decide based on its own
- * bandwidth limit detection.
- */
-const simulcastVideoEncodings: RTCRtpEncodingParameters[] = [
-    {
-        rid: "r0",
-        scaleResolutionDownBy: 4,
-    },
-    {
-        rid: "r1",
-        scaleResolutionDownBy: 2,
-    },
-    {
-        rid: "r2",
-        scaleResolutionDownBy: 1,
-    },
-];
-
 export interface WebRtcPeerConfiguration {
     mediaConstraints: {
-        audio: boolean,
+        audio: boolean;
-        video: boolean
+        video: boolean;
     };
     simulcast: boolean;
     mediaServer: string;
@@ -105,7 +44,7 @@ export interface WebRtcPeerConfiguration {
     mediaStream?: MediaStream | null;
     mode?: 'sendonly' | 'recvonly' | 'sendrecv';
     id?: string;
-    typeOfVideo: TypeOfVideo | undefined
+    typeOfVideo: TypeOfVideo | undefined;
 }
 
 export class WebRtcPeer {
@@ -135,8 +74,9 @@ export class WebRtcPeer {
                     : null,
             mode: !!configuration.mode ? configuration.mode : "sendrecv",
             id: !!configuration.id ? configuration.id : this.generateUniqueId(),
-            typeOfVideo: configuration.typeOfVideo
         };
+        // prettier-ignore
+        logger.debug(`[WebRtcPeer] configuration:\n${JSON.stringify(this.configuration, null, 2)}`);
 
         this.pc = new RTCPeerConnection({ iceServers: this.configuration.iceServers });
 
@@ -207,11 +147,111 @@ export class WebRtcPeer {
                             direction: this.configuration.mode,
                             streams: [this.configuration.mediaStream],
                         };
-                        if (this.configuration.simulcast && track.kind === "video") {
+
-                            tcInit.sendEncodings = simulcastVideoEncodings;
+                        if (track.kind === "audio") {
+                            if ("contentHint" in track) {
+                                // For audio: "", "speech", "speech-recognition", "music".
+                                // https://w3c.github.io/mst-content-hint/#audio-content-hints
+                                track.contentHint = "";
+                                logger.info(`[createOffer] Audio track Content Hint set: '${track.contentHint}'`);
                             }
+                        } else if (track.kind === "video") {
+                            if ("contentHint" in track) {
+                                // For video: "", "motion", "detail", "text".
+                                // https://w3c.github.io/mst-content-hint/#video-content-hints
+                                if (this.configuration.typeOfVideo === TypeOfVideo.SCREEN) {
+                                    track.contentHint = "detail";
+                                } else {
+                                    track.contentHint = "motion";
+                                }
+                                logger.info(`[createOffer] Video track Content Hint set: '${track.contentHint}'`);
+                            }
+
+                            logger.info(`[createOffer] this.configuration.simulcast: ${this.configuration.simulcast}`);
+                            if (this.configuration.simulcast) {
+                                // Check if the requested size is enough to ask for 3 layers.
+                                const trackSettings = track.getSettings();
+                                const trackConsts = track.getConstraints();
+
+                                const trackWidth: number =
+                                    trackSettings.width ??
+                                    (trackConsts.width as ConstrainULongRange).ideal ??
+                                    (trackConsts.width as number) ??
+                                    0;
+                                const trackHeight: number =
+                                    trackSettings.height ??
+                                    (trackConsts.height as ConstrainULongRange).ideal ??
+                                    (trackConsts.height as number) ??
+                                    0;
+                                logger.info(`[createOffer] Video track dimensions: ${trackWidth}x${trackHeight}`);
+
+                                const trackPixels = trackWidth * trackHeight;
+                                let maxLayers = 0;
+                                if (trackPixels >= 960 * 540) {
+                                    maxLayers = 3;
+                                } else if (trackPixels >= 480 * 270) {
+                                    maxLayers = 2;
+                                } else {
+                                    maxLayers = 1;
+                                }
+
+                                tcInit.sendEncodings = [];
+                                for (let l = 0; l < maxLayers; l++) {
+                                    const layerDiv = 2 ** (maxLayers - l - 1);
+
+                                    const encoding: RTCRtpEncodingParameters = {
+                                        rid: "rDiv" + layerDiv.toString(),
+
+                                        // @ts-ignore: Property missing from DOM types.
+                                        scalabilityMode: "L1T1",
+                                    };
+
+                                    if (this.configuration.typeOfVideo === TypeOfVideo.SCREEN) {
+                                        // Prioritize best resolution, for maximum picture detail.
+                                        encoding.scaleResolutionDownBy = 1.0;
+
+                                        // @ts-ignore: Property missing from DOM types.
+                                        encoding.maxFramerate = Math.floor(30 / layerDiv);
+                                        // encoding.maxFramerate = (l === 2) ? 30 : Math.floor(30 / (2 * layerDiv)); // TESTING
+                                    } else {
+                                        encoding.scaleResolutionDownBy = layerDiv;
+                                    }
+
+                                    tcInit.sendEncodings.push(encoding);
+                                }
+                            }
+                        }
+
                         const tc = this.pc.addTransceiver(track, tcInit);
 
+                        if (track.kind === "video") {
+                            let sendParams = tc.sender.getParameters();
+                            let needSetParams = false;
+
+                            if (!("degradationPreference" in sendParams)) {
+                                logger.debug(
+                                    `[createOffer] RTCRtpSendParameters.degradationPreference attribute not present`
+                                );
+                                // Asked about why this might happen. Check it:
+                                // https://groups.google.com/g/discuss-webrtc/c/R8Xug-irfRY
+
+                                // For video: "balanced", "maintain-framerate", "maintain-resolution".
+                                if (this.configuration.typeOfVideo === TypeOfVideo.SCREEN) {
+                                    sendParams.degradationPreference = "maintain-resolution";
+                                } else {
+                                    sendParams.degradationPreference = "balanced";
+                                }
+
+                                logger.debug(
+                                    `[createOffer] Video sender Degradation Preference set: ${sendParams.degradationPreference}`
+                                );
+
+                                // FIXME: Firefox implements degradationPreference on each individual encoding!
+                                // (set it on every element of the sendParams.encodings array)
+
+                                needSetParams = true;
+                            }
+
                             // FIXME: Check that the simulcast encodings were applied.
                             // Firefox doesn't implement `RTCRtpTransceiverInit.sendEncodings`
                             // so the only way to enable simulcast is with `RTCRtpSender.setParameters()`.
@@ -222,17 +262,38 @@ export class WebRtcPeer {
                             // NOTE: This is done in a way that is compatible with all browsers, to save on
                             // browser-conditional code. The idea comes from WebRTC Adapter.js:
                             // * https://github.com/webrtcHacks/adapter/issues/998
-                        // * https://github.com/webrtcHacks/adapter/blob/845a3b4874f1892a76f04c3cc520e80b5041c303/src/js/firefox/firefox_shim.js#L217
+                            // * https://github.com/webrtcHacks/adapter/blob/v7.7.0/src/js/firefox/firefox_shim.js#L231-L255
-                        if (this.configuration.simulcast && track.kind === "video") {
+                            if (this.configuration.simulcast) {
-                            const sendParams = tc.sender.getParameters();
                                 if (
                                     !("encodings" in sendParams) ||
                                     sendParams.encodings.length !== tcInit.sendEncodings!.length
                                 ) {
                                     sendParams.encodings = tcInit.sendEncodings!;
+
+                                    needSetParams = true;
+                                }
+                            }
+
+                            if (needSetParams) {
+                                logger.debug(`[createOffer] Setting new RTCRtpSendParameters`);
                                 await tc.sender.setParameters(sendParams);
                             }
                         }
+
+                        // DEBUG: Uncomment for details.
+                        // if (track.kind === "video" && this.configuration.simulcast) {
+                        //     // Print browser capabilities.
+                        //     // prettier-ignore
+                        //     logger.error(`[createOffer] Transceiver send capabilities (static):\n${JSON.stringify(RTCRtpSender.getCapabilities?.("video"), null, 2)}`);
+                        //     // prettier-ignore
+                        //     logger.error(`[createOffer] Transceiver recv capabilities (static):\n${JSON.stringify(RTCRtpReceiver.getCapabilities?.("video"), null, 2)}`);
+
+                        //     // Print requested Transceiver encodings and parameters.
+                        //     // prettier-ignore
+                        //     logger.error(`[createOffer] Transceiver send encodings (requested):\n${JSON.stringify(tcInit.sendEncodings, null, 2)}`);
+                        //     // prettier-ignore
+                        //     logger.error(`[createOffer] Transceiver send parameters (requested):\n${JSON.stringify(tc.sender.getParameters(), null, 2)}`);
+                        // }
                     }
                 } else {
                     // To just receive media, create new recvonly transceivers.
@@ -427,8 +488,18 @@ export class WebRtcPeer {
                 return reject('RTCPeerConnection is closed when trying to set remote description');
             }
             this.setRemoteDescription(answer)
-                .then(() => resolve())
+                .then(() => {
-                .catch(error => reject(error));
+                    // DEBUG: Uncomment for details.
+                    // {
+                    //     const tc = this.pc.getTransceivers().find((tc) => tc.sender.track?.kind === "video");
+                    //     const sendParams = tc?.sender.getParameters();
+                    //     // prettier-ignore
+                    //     logger.error(`[processRemoteAnswer] Transceiver send parameters (effective):\n${JSON.stringify(sendParams, null, 2)}`);
+                    // }
+
+                    resolve();
+                })
+                .catch((error) => reject(error));
         });
     }