openvidu-browser: OpenVidu.getUserMedia fix. generateMediaConstraints refactoring

openvidu-browser/src/OpenVidu/OpenVidu.ts

@@ -23,6 +23,7 @@ import { StreamPropertyChangedEvent } from '../OpenViduInternal/Events/StreamPro
 import { Device } from '../OpenViduInternal/Interfaces/Public/Device';
 import { OpenViduAdvancedConfiguration } from '../OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration';
 import { PublisherProperties } from '../OpenViduInternal/Interfaces/Public/PublisherProperties';
+import { CustomMediaStreamConstraints } from '../OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints';
 import { OpenViduError, OpenViduErrorName } from '../OpenViduInternal/Enums/OpenViduError';
 import { VideoInsertMode } from '../OpenViduInternal/Enums/VideoInsertMode';
 
@@ -551,20 +552,41 @@ export class OpenVidu {
           });
       }
 
-      this.generateMediaConstraints(options)
+      this.generateMediaConstraints(options).then(myConstraints => {
-        .then(constraints => {
+
+        if (!!myConstraints.videoTrack && !!myConstraints.audioTrack ||
+          !!myConstraints.audioTrack && myConstraints.constraints?.video === false ||
+          !!myConstraints.videoTrack && myConstraints.constraints?.audio === false) {
+
+          // No need to call getUserMedia at all. Both tracks provided, or only AUDIO track provided or only VIDEO track provided
+          resolve(this.addAlreadyProvidedTracks(myConstraints, new MediaStream()));
+
+        } else {
+          // getUserMedia must be called. AUDIO or VIDEO are requesting a new track
+
+          // Delete already provided constraints for audio or video
+          if (!!myConstraints.videoTrack) {
+            delete myConstraints.constraints!.video;
+          }
+          if (!!myConstraints.audioTrack) {
+            delete myConstraints.constraints!.audio;
+          }
+
           let mustAskForAudioTrackLater = false;
           if (typeof options.videoSource === 'string') {
+            // Video is deviceId or screen sharing
             if (options.videoSource === 'screen' ||
               options.videoSource === 'window' ||
               (platform.name === 'Electron' && options.videoSource.startsWith('screen:'))) {
-              // Screen sharing
+              // Video is screen sharing
-              mustAskForAudioTrackLater = options.audioSource !== null && options.audioSource !== false;
+              mustAskForAudioTrackLater = !myConstraints.audioTrack && (options.audioSource !== null && options.audioSource !== false);
               if (navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') {
+                // getDisplayMedia supported
                 navigator.mediaDevices['getDisplayMedia']({ video: true })
                   .then(mediaStream => {
+                    this.addAlreadyProvidedTracks(myConstraints, mediaStream);
                     if (mustAskForAudioTrackLater) {
-                      askForAudioStreamOnly(mediaStream, constraints);
+                      askForAudioStreamOnly(mediaStream, <MediaStreamConstraints>myConstraints.constraints);
                       return;
                     } else {
                       resolve(mediaStream);
@@ -575,15 +597,21 @@ export class OpenVidu {
                     const errorMessage = error.toString();
                     reject(new OpenViduError(errorName, errorMessage));
                   });
+                return;
+              } else {
+                // getDisplayMedia NOT supported. Can perform getUserMedia below with already calculated constraints
               }
-              return;
+            } else {
+              // Video is deviceId. Can perform getUserMedia below with already calculated constraints
             }
           }
-          const constraintsAux = mustAskForAudioTrackLater ? { video: constraints.video } : constraints;
+          // Use already calculated constraints
+          const constraintsAux = mustAskForAudioTrackLater ? { video: myConstraints.constraints!.video } : myConstraints.constraints;
           navigator.mediaDevices.getUserMedia(constraintsAux)
             .then(mediaStream => {
+              this.addAlreadyProvidedTracks(myConstraints, mediaStream);
               if (mustAskForAudioTrackLater) {
-                askForAudioStreamOnly(mediaStream, constraints);
+                askForAudioStreamOnly(mediaStream, <MediaStreamConstraints>myConstraints.constraints);
                 return;
               } else {
                 resolve(mediaStream);
@@ -599,10 +627,10 @@ export class OpenVidu {
               }
               reject(new OpenViduError(errorName, errorMessage));
             });
-        })
+        }
-        .catch((error: OpenViduError) => {
+      }).catch((error: OpenViduError) => {
-          reject(error);
+        reject(error);
-        });
+      });
     });
   }
 
@@ -638,168 +666,192 @@ export class OpenVidu {
   /**
    * @hidden
    */
-  generateMediaConstraints(publisherProperties: PublisherProperties): Promise<MediaStreamConstraints> {
+  generateMediaConstraints(publisherProperties: PublisherProperties): Promise<CustomMediaStreamConstraints> {
-    return new Promise<MediaStreamConstraints>((resolve, reject) => {
+    return new Promise<CustomMediaStreamConstraints>((resolve, reject) => {
-      let audio, video;
 
-      if (publisherProperties.audioSource === null || publisherProperties.audioSource === false) {
+      const myConstraints: CustomMediaStreamConstraints = {
-        audio = false;
+        audioTrack: undefined,
-      } else if (publisherProperties.audioSource === undefined) {
+        videoTrack: undefined,
-        audio = true;
+        constraints: {
-      } else {
+          audio: undefined,
-        audio = publisherProperties.audioSource;
+          video: undefined
+        }
       }
+      const audioSource = publisherProperties.audioSource;
+      const videoSource = publisherProperties.videoSource;
 
-      if (publisherProperties.videoSource === null || publisherProperties.videoSource === false) {
+      // CASE 1: null/false
-        video = false;
+      if (audioSource === null || audioSource === false) {
-      } else {
+        // No audio track
-        video = {
+        myConstraints.constraints!.audio = false;
-          height: {
-            ideal: 480
-          },
-          width: {
-            ideal: 640
-          }
-        };
       }
-
+      if (videoSource === null || videoSource === false) {
-      if (audio === false && video === false) {
+        // No video track
+        myConstraints.constraints!.video = false;
+      }
+      if (myConstraints.constraints!.audio === false && myConstraints.constraints!.video === false) {
+        // ERROR! audioSource and videoSource cannot be both false at the same time
         reject(new OpenViduError(OpenViduErrorName.NO_INPUT_SOURCE_SET,
           "Properties 'audioSource' and 'videoSource' cannot be set to false or null at the same time"));
       }
 
-      const mediaConstraints: MediaStreamConstraints = {
+      // CASE 2: MediaStreamTracks
-        audio,
+      if (typeof MediaStreamTrack !== 'undefined' && audioSource instanceof MediaStreamTrack) {
-        video
+        // Already provided audio track
-      };
+        myConstraints.audioTrack = audioSource;
-
+      }
-      if (typeof mediaConstraints.audio === 'string') {
+      if (typeof MediaStreamTrack !== 'undefined' && videoSource instanceof MediaStreamTrack) {
-        mediaConstraints.audio = { deviceId: { exact: mediaConstraints.audio } };
+        // Already provided video track
+        myConstraints.videoTrack = videoSource;
       }
 
-      if (mediaConstraints.video) {
+      // CASE 3: Default tracks
+      if (audioSource === undefined) {
+        myConstraints.constraints!.audio = true;
+      }
+      if (videoSource === undefined) {
+        myConstraints.constraints!.video = {
+          width: {
+            ideal: 640
+          },
+          height: {
+            ideal: 480
+          }
+        };
+      }
 
+      // CASE 3.5: give values to resolution and frameRate if video not null/false
+      if (videoSource !== null && videoSource !== false) {
         if (!!publisherProperties.resolution) {
           const widthAndHeight = publisherProperties.resolution.toLowerCase().split('x');
-          const width = Number(widthAndHeight[0]);
+          const idealWidth = Number(widthAndHeight[0]);
-          const height = Number(widthAndHeight[1]);
+          const idealHeight = Number(widthAndHeight[1]);
-          (mediaConstraints.video as any).width.ideal = width;
+          myConstraints.constraints!.video = {
-          (mediaConstraints.video as any).height.ideal = height;
+            width: {
+              ideal: idealWidth
+            },
+            height: {
+              ideal: idealHeight
+            }
+          }
         }
-
         if (!!publisherProperties.frameRate) {
-          (mediaConstraints.video as any).frameRate = { ideal: publisherProperties.frameRate };
+          (<MediaTrackConstraints>myConstraints.constraints!.video).frameRate = { ideal: publisherProperties.frameRate };
         }
+      }
 
-        if (!!publisherProperties.videoSource && typeof publisherProperties.videoSource === 'string') {
+      // CASE 4: deviceId or screen sharing
+      if (typeof audioSource === 'string') {
+        myConstraints.constraints!.audio = { deviceId: { exact: audioSource } };
+      }
+      if (typeof videoSource === 'string') {
 
-          if (publisherProperties.videoSource === 'screen' ||
+        if (!this.isScreenShare(videoSource)) {
-            publisherProperties.videoSource === 'window' ||
+          if (!myConstraints.constraints!.video) {
-            (platform.name === 'Electron' && publisherProperties.videoSource.startsWith('screen:'))) {
+            myConstraints.constraints!.video = {};
+          }
+          (<MediaTrackConstraints>myConstraints.constraints!.video)['deviceId'] = { exact: videoSource };
+        } else {
 
-            if (!this.checkScreenSharingCapabilities()) {
+          // Screen sharing
 
-              const error = new OpenViduError(OpenViduErrorName.SCREEN_SHARING_NOT_SUPPORTED, 'You can only screen share in desktop Chrome, Firefox, Opera or Electron. Detected client: ' + platform.name);
+          if (!this.checkScreenSharingCapabilities()) {
-              console.error(error);
+            const error = new OpenViduError(OpenViduErrorName.SCREEN_SHARING_NOT_SUPPORTED, 'You can only screen share in desktop Chrome, Firefox, Opera or Electron. Detected client: ' + platform.name);
-              reject(error);
+            console.error(error);
+            reject(error);
+          } else {
+
+            if (platform.name === 'Electron') {
+              const prefix = "screen:";
+              const videoSourceString: string = videoSource;
+              const electronScreenId = videoSourceString.substr(videoSourceString.indexOf(prefix) + prefix.length);
+              (<any>myConstraints.constraints!.video) = {
+                mandatory: {
+                  chromeMediaSource: 'desktop',
+                  chromeMediaSourceId: electronScreenId
+                }
+              };
+              resolve(myConstraints);
 
             } else {
 
-              if (platform.name === 'Electron') {
+              if (!!this.advancedConfiguration.screenShareChromeExtension && !(platform.name!.indexOf('Firefox') !== -1) && !navigator.mediaDevices['getDisplayMedia']) {
 
-                const prefix = "screen:";
+                // Custom screen sharing extension for Chrome (and Opera) and no support for MediaDevices.getDisplayMedia()
-                const videoSourceString: string = publisherProperties.videoSource;
-                const electronScreenId = videoSourceString.substr(videoSourceString.indexOf(prefix) + prefix.length);
-                (<any>mediaConstraints['video']) = {
-                  mandatory: {
-                    chromeMediaSource: 'desktop',
-                    chromeMediaSourceId: electronScreenId
-                  }
-                };
-                resolve(mediaConstraints);
 
-              } else {
+                screenSharing.getScreenConstraints((error, screenConstraints) => {
-
+                  if (!!error || !!screenConstraints.mandatory && screenConstraints.mandatory.chromeMediaSource === 'screen') {
-                if (!!this.advancedConfiguration.screenShareChromeExtension && !(platform.name!.indexOf('Firefox') !== -1) && !navigator.mediaDevices['getDisplayMedia']) {
+                    if (error === 'permission-denied' || error === 'PermissionDeniedError') {
-
+                      const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop');
-                  // Custom screen sharing extension for Chrome (and Opera) and no support for MediaDevices.getDisplayMedia()
+                      console.error(error);
-
+                      reject(error);
-                  screenSharing.getScreenConstraints((error, screenConstraints) => {
-                    if (!!error || !!screenConstraints.mandatory && screenConstraints.mandatory.chromeMediaSource === 'screen') {
-                      if (error === 'permission-denied' || error === 'PermissionDeniedError') {
-                        const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop');
-                        console.error(error);
-                        reject(error);
-                      } else {
-                        const extensionId = this.advancedConfiguration.screenShareChromeExtension!.split('/').pop()!!.trim();
-                        screenSharing.getChromeExtensionStatus(extensionId, (status) => {
-                          if (status === 'installed-disabled') {
-                            const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension');
-                            console.error(error);
-                            reject(error);
-                          }
-                          if (status === 'not-installed') {
-                            const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, (<string>this.advancedConfiguration.screenShareChromeExtension));
-                            console.error(error);
-                            reject(error);
-                          }
-                        });
-                      }
                     } else {
-                      mediaConstraints.video = screenConstraints;
+                      const extensionId = this.advancedConfiguration.screenShareChromeExtension!.split('/').pop()!!.trim();
-                      resolve(mediaConstraints);
+                      screenSharing.getChromeExtensionStatus(extensionId, status => {
-                    }
+                        if (status === 'installed-disabled') {
-                  });
-
-                } else {
-
-                  if (navigator.mediaDevices['getDisplayMedia']) {
-                    // getDisplayMedia support (Chrome >= 72, Firefox >= 66)
-                    resolve(mediaConstraints);
-                  } else {
-                    // Default screen sharing extension for Chrome/Opera, or is Firefox < 66
-                    const firefoxString = platform.name!.indexOf('Firefox') !== -1 ? publisherProperties.videoSource : undefined;
-
-                    screenSharingAuto.getScreenId(firefoxString, (error, sourceId, screenConstraints) => {
-                      if (!!error) {
-                        if (error === 'not-installed') {
-                          const extensionUrl = !!this.advancedConfiguration.screenShareChromeExtension ? this.advancedConfiguration.screenShareChromeExtension :
-                            'https://chrome.google.com/webstore/detail/openvidu-screensharing/lfcgfepafnobdloecchnfaclibenjold';
-                          const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, extensionUrl);
-                          console.error(error);
-                          reject(error);
-                        } else if (error === 'installed-disabled') {
                           const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension');
                           console.error(error);
                           reject(error);
-                        } else if (error === 'permission-denied') {
+                        }
-                          const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop');
+                        if (status === 'not-installed') {
+                          const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, (<string>this.advancedConfiguration.screenShareChromeExtension));
                           console.error(error);
                           reject(error);
                         }
-                      } else {
+                      });
-                        mediaConstraints.video = screenConstraints.video;
+                      return;
-                        resolve(mediaConstraints);
+                    }
-                      }
+                  } else {
-                    });
+                    myConstraints.constraints!.video = screenConstraints;
+                    resolve(myConstraints);
                   }
+                });
+                return;
+              } else {
+
+                if (navigator.mediaDevices['getDisplayMedia']) {
+                  // getDisplayMedia support (Chrome >= 72, Firefox >= 66)
+                  resolve(myConstraints);
+                } else {
+                  // Default screen sharing extension for Chrome/Opera, or is Firefox < 66
+                  const firefoxString = platform.name!.indexOf('Firefox') !== -1 ? publisherProperties.videoSource : undefined;
+
+                  screenSharingAuto.getScreenId(firefoxString, (error, sourceId, screenConstraints) => {
+                    if (!!error) {
+                      if (error === 'not-installed') {
+                        const extensionUrl = !!this.advancedConfiguration.screenShareChromeExtension ? this.advancedConfiguration.screenShareChromeExtension :
+                          'https://chrome.google.com/webstore/detail/openvidu-screensharing/lfcgfepafnobdloecchnfaclibenjold';
+                        const err = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, extensionUrl);
+                        console.error(err);
+                        reject(err);
+                      } else if (error === 'installed-disabled') {
+                        const err = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension');
+                        console.error(err);
+                        reject(err);
+                      } else if (error === 'permission-denied') {
+                        const err = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop');
+                        console.error(err);
+                        reject(err);
+                      } else {
+                        const err = new OpenViduError(OpenViduErrorName.GENERIC_ERROR, 'Unknown error when accessing screen share');
+                        console.error(err);
+                        console.error(error);
+                        reject(err);
+                      }
+                    } else {
+                      myConstraints.constraints!.video = screenConstraints.video;
+                      resolve(myConstraints);
+                    }
+                  });
+                  return;
                 }
-
-                publisherProperties.videoSource = 'screen';
-
               }
             }
-          } else {
-            // tslint:disable-next-line:no-string-literal
-            mediaConstraints.video['deviceId'] = { exact: publisherProperties.videoSource };
-            resolve(mediaConstraints);
           }
-        } else {
-          resolve(mediaConstraints);
         }
-      } else {
-        resolve(mediaConstraints);
       }
+
+      resolve(myConstraints);
     });
   }
 
@@ -912,6 +964,18 @@ export class OpenVidu {
     }
   }
 
+  /**
+   * @hidden
+   */
+  addAlreadyProvidedTracks(myConstraints: CustomMediaStreamConstraints, mediaStream: MediaStream) {
+    if (!!myConstraints.videoTrack) {
+      mediaStream.addTrack(myConstraints.videoTrack);
+    }
+    if (!!myConstraints.audioTrack) {
+      mediaStream.addTrack(myConstraints.audioTrack);
+    }
+    return mediaStream;
+  }
 
   /* Private methods */
 
@@ -961,4 +1025,10 @@ export class OpenVidu {
     }
   }
 
+  private isScreenShare(videoSource: string) {
+    return videoSource === 'screen' ||
+      videoSource === 'window' ||
+      (platform.name === 'Electron' && videoSource.startsWith('screen:'))
+  }
+
 }

openvidu-browser/src/OpenVidu/Publisher.ts

@@ -279,7 +279,22 @@ export class Publisher extends StreamManager {
      */
     replaceTrack(track: MediaStreamTrack): Promise<any> {
         return new Promise((resolve, reject) => {
-            this.stream.getRTCPeerConnection().getSenders()[0].replaceTrack(track).then(() => {
+            const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders();
+            let sender: RTCRtpSender | undefined;
+            if (track.kind === 'video') {
+                sender = senders.find(s => !!s.track && s.track.kind === 'video');
+                if (!sender) {
+                    reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'))
+                }
+            } else if (track.kind === 'audio') {
+                sender = senders.find(s => !!s.track && s.track.kind === 'audio');
+                if (!sender) {
+                    reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'))
+                }
+            } else {
+                reject(new Error('Unknown track kind ' + track.kind));
+            }
+            (<any>sender).replaceTrack(track).then(() => {
                 let removedTrack: MediaStreamTrack;
                 if (track.kind === 'video') {
                     removedTrack = this.stream.getMediaStream().getVideoTracks()[0];
@@ -567,30 +582,17 @@ export class Publisher extends StreamManager {
                 }
             }
 
-            // Check if new constraints need to be generated. No constraints needed if
-            // - video track is given and no audio
-            // - audio track is given and no video
-            // - both video and audio tracks are given
-            if ((typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack && !this.properties.audioSource)
-                || (typeof MediaStreamTrack !== 'undefined' && this.properties.audioSource instanceof MediaStreamTrack && !this.properties.videoSource)
-                || (typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack && this.properties.audioSource instanceof MediaStreamTrack)) {
-                const mediaStream = new MediaStream();
-                if (typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack) {
-                    mediaStream.addTrack((<MediaStreamTrack>this.properties.videoSource));
-                }
-                if (typeof MediaStreamTrack !== 'undefined' && this.properties.audioSource instanceof MediaStreamTrack) {
-                    mediaStream.addTrack((<MediaStreamTrack>this.properties.audioSource));
-                }
-                // MediaStreamTracks are handled within callback - just call callback with new MediaStream() and let it handle the sources
-                successCallback(mediaStream);
-                // Return as we do not need to process further
-                return;
-            }
-
             this.openvidu.generateMediaConstraints(this.properties)
                 .then(myConstraints => {
 
-                    constraints = myConstraints;
+                    if (myConstraints.constraints === undefined) {
+                        // No need to call getUserMedia at all. MediaStreamTracks already provided
+                        successCallback(this.openvidu.addAlreadyProvidedTracks(myConstraints, new MediaStream()));
+                        // Return as we do not need to process further
+                        return;
+                    }
+
+                    constraints = myConstraints.constraints;
 
                     const outboundStreamOptions = {
                         mediaConstraints: constraints,
@@ -605,19 +607,18 @@ export class Publisher extends StreamManager {
                     this.setPermissionDialogTimer(timeForDialogEvent);
 
                     if (this.stream.isSendScreen() && navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') {
-
                         navigator.mediaDevices['getDisplayMedia']({ video: true })
                             .then(mediaStream => {
+                                this.openvidu.addAlreadyProvidedTracks(myConstraints, mediaStream);
                                 getMediaSuccess(mediaStream, definedAudioConstraint);
                             })
                             .catch(error => {
                                 getMediaError(error);
                             });
-
                     } else {
-
                         navigator.mediaDevices.getUserMedia(constraintsAux)
                             .then(mediaStream => {
+                                this.openvidu.addAlreadyProvidedTracks(myConstraints, mediaStream);
                                 getMediaSuccess(mediaStream, definedAudioConstraint);
                             })
                             .catch(error => {

openvidu-browser/src/OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints.ts

@@ -0,0 +1,22 @@
+/*
+ * (C) Copyright 2017-2020 OpenVidu (https://openvidu.io)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+export interface CustomMediaStreamConstraints {
+    constraints: MediaStreamConstraints | undefined;
+    audioTrack: MediaStreamTrack | undefined;
+    videoTrack: MediaStreamTrack | undefined;
+}