openvidu-browser: OpenVidu.getUserMedia works for screen videoSource

openvidu-browser/src/OpenVidu/OpenVidu.ts

@@ -521,11 +521,61 @@ export class OpenVidu {
    */
   getUserMedia(options: PublisherProperties): Promise<MediaStream> {
     return new Promise<MediaStream>((resolve, reject) => {
+
+      const askForAudioStreamOnly = (previousMediaStream: MediaStream, constraints: MediaStreamConstraints) => {
+        const definedAudioConstraint = ((constraints.audio === undefined) ? true : constraints.audio);
+        const constraintsAux: MediaStreamConstraints = { audio: definedAudioConstraint, video: false };
+        navigator.mediaDevices.getUserMedia(constraintsAux)
+          .then(audioOnlyStream => {
+            previousMediaStream.addTrack(audioOnlyStream.getAudioTracks()[0]);
+            resolve(previousMediaStream);
+          })
+          .catch(error => {
+            previousMediaStream.getAudioTracks().forEach((track) => {
+              track.stop();
+            });
+            previousMediaStream.getVideoTracks().forEach((track) => {
+              track.stop();
+            });
+            reject(this.generateAudioDeviceError(error, constraintsAux));
+          });
+      }
+
       this.generateMediaConstraints(options)
         .then(constraints => {
-          navigator.mediaDevices.getUserMedia(constraints)
+          let mustAskForAudioTrackLater = false;
+          if (typeof options.videoSource === 'string') {
+            if (options.videoSource === 'screen' ||
+              (platform.name!.indexOf('Firefox') !== -1 && options.videoSource === 'window')) {
+              // Screen sharing
+              mustAskForAudioTrackLater = options.audioSource !== null && options.audioSource !== false;
+              if (navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') {
+                navigator.mediaDevices['getDisplayMedia']({ video: true })
+                  .then(mediaStream => {
+                    if (mustAskForAudioTrackLater) {
+                      askForAudioStreamOnly(mediaStream, constraints);
+                      return;
+                    } else {
+                      resolve(mediaStream);
+                    }
+                  })
+                  .catch(error => {
+                    let errorName: OpenViduErrorName = OpenViduErrorName.SCREEN_CAPTURE_DENIED;
+                    const errorMessage = error.toString();
+                    reject(new OpenViduError(errorName, errorMessage));
+                  });
+              }
+            }
+          }
+          const constraintsAux = mustAskForAudioTrackLater ? { video: constraints.video } : constraints;
+          navigator.mediaDevices.getUserMedia(constraintsAux)
             .then(mediaStream => {
-              resolve(mediaStream);
+              if (mustAskForAudioTrackLater) {
+                askForAudioStreamOnly(mediaStream, constraints);
+                return;
+              } else {
+                resolve(mediaStream);
+              }
             })
             .catch(error => {
               let errorName: OpenViduErrorName;
@@ -599,6 +649,11 @@ export class OpenVidu {
         };
       }
 
+      if (audio === false && video === false) {
+        reject(new OpenViduError(OpenViduErrorName.NO_INPUT_SOURCE_SET,
+          "Properties 'audioSource' and 'videoSource' cannot be set to false or null at the same time"));
+      }
+
       const mediaConstraints: MediaStreamConstraints = {
         audio,
         video
@@ -808,6 +863,42 @@ export class OpenVidu {
     return this.recorder;
   }
 
+  /**
+   * @hidden
+   */
+  generateAudioDeviceError(error, constraints: MediaStreamConstraints): OpenViduError {
+    if (error.name === 'Error') {
+      // Safari OverConstrainedError has as name property 'Error' instead of 'OverConstrainedError'
+      error.name = error.constructor.name;
+    }
+    let errorName, errorMessage: string;
+    switch (error.name.toLowerCase()) {
+      case 'notfounderror':
+        errorName = OpenViduErrorName.INPUT_AUDIO_DEVICE_NOT_FOUND;
+        errorMessage = error.toString();
+        return new OpenViduError(errorName, errorMessage);
+      case 'notallowederror':
+        errorName = OpenViduErrorName.DEVICE_ACCESS_DENIED;
+        errorMessage = error.toString();
+        return new OpenViduError(errorName, errorMessage);
+      case 'overconstrainederror':
+        if (error.constraint.toLowerCase() === 'deviceid') {
+          errorName = OpenViduErrorName.INPUT_AUDIO_DEVICE_NOT_FOUND;
+          errorMessage = "Audio input device with deviceId '" + (<ConstrainDOMStringParameters>(<MediaTrackConstraints>constraints.audio).deviceId!!).exact + "' not found";
+        } else {
+          errorName = OpenViduErrorName.PUBLISHER_PROPERTIES_ERROR;
+          errorMessage = "Audio input device doesn't support the value passed for constraint '" + error.constraint + "'";
+        }
+        return new OpenViduError(errorName, errorMessage);
+      case 'notreadableerror':
+        errorName = OpenViduErrorName.DEVICE_ALREADY_IN_USE;
+        errorMessage = error.toString();
+        return (new OpenViduError(errorName, errorMessage));
+      default:
+        return new OpenViduError(OpenViduErrorName.INPUT_AUDIO_DEVICE_GENERIC_ERROR, error.toString());
+    }
+  }
+
 
   /* Private methods */
 

openvidu-browser/src/OpenVidu/Publisher.ts

@@ -270,8 +270,7 @@ export class Publisher extends StreamManager {
      * 
      * You can get this new MediaStreamTrack by using the native Web API or simply with [[OpenVidu.getUserMedia]] method.
      * 
-     * **WARNING: this method has been proven to work, but there may be some combinations of published/replaced tracks that may be incompatible between them and break the connection in OpenVidu Server.**
+     * **WARNING: this method has been proven to work, but there may be some combinations of published/replaced tracks that may be incompatible between them and break the connection in OpenVidu Server. A complete renegotiation may be the only solution in this case**
-     * **A complete renegotiation may be the only solution in this case**
      * 
      * @param track The [MediaStreamTrack](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) object to replace the current one. If it is an audio track, the current audio track will be the replaced one. If it
      * is a video track, the current video track will be the replaced one.
@@ -281,6 +280,15 @@ export class Publisher extends StreamManager {
     replaceTrack(track: MediaStreamTrack): Promise<any> {
         return new Promise((resolve, reject) => {
             this.stream.getRTCPeerConnection().getSenders()[0].replaceTrack(track).then(() => {
+                let removedTrack: MediaStreamTrack;
+                if (track.kind === 'video') {
+                    removedTrack = this.stream.getMediaStream().getVideoTracks()[0];
+                } else {
+                    removedTrack = this.stream.getMediaStream().getAudioTracks()[0];
+                }
+                this.stream.getMediaStream().removeTrack(removedTrack);
+                removedTrack.stop();
+                this.stream.getMediaStream().addTrack(track);
                 resolve();
             }).catch(error => {
                 reject(error);
@@ -471,33 +479,14 @@ export class Publisher extends StreamManager {
                         })
                         .catch(error => {
                             this.clearPermissionDialogTimer(startTime, timeForDialogEvent);
-                            if (error.name === 'Error') {
+                            mediaStream.getAudioTracks().forEach((track) => {
-                                // Safari OverConstrainedError has as name property 'Error' instead of 'OverConstrainedError'
+                                track.stop();
-                                error.name = error.constructor.name;
+                            });
-                            }
+                            mediaStream.getVideoTracks().forEach((track) => {
-                            let errorName, errorMessage;
+                                track.stop();
-                            switch (error.name.toLowerCase()) {
+                            });
-                                case 'notfounderror':
+                            errorCallback(this.openvidu.generateAudioDeviceError(error, constraints));
-                                    errorName = OpenViduErrorName.INPUT_AUDIO_DEVICE_NOT_FOUND;
+                            return;
-                                    errorMessage = error.toString();
-                                    errorCallback(new OpenViduError(errorName, errorMessage));
-                                    break;
-                                case 'notallowederror':
-                                    errorName = OpenViduErrorName.DEVICE_ACCESS_DENIED;
-                                    errorMessage = error.toString();
-                                    errorCallback(new OpenViduError(errorName, errorMessage));
-                                    break;
-                                case 'overconstrainederror':
-                                    if (error.constraint.toLowerCase() === 'deviceid') {
-                                        errorName = OpenViduErrorName.INPUT_AUDIO_DEVICE_NOT_FOUND;
-                                        errorMessage = "Audio input device with deviceId '" + (<ConstrainDOMStringParameters>(<MediaTrackConstraints>constraints.video).deviceId!!).exact + "' not found";
-                                    } else {
-                                        errorName = OpenViduErrorName.PUBLISHER_PROPERTIES_ERROR;
-                                        errorMessage = "Audio input device doesn't support the value passed for constraint '" + error.constraint + "'";
-                                    }
-                                    errorCallback(new OpenViduError(errorName, errorMessage));
-                                    break;
-                            }
                         });
                 } else {
                     successCallback(mediaStream);
@@ -607,41 +596,35 @@ export class Publisher extends StreamManager {
                         mediaConstraints: constraints,
                         publisherProperties: this.properties
                     };
-
                     this.stream.setOutboundStreamOptions(outboundStreamOptions);
 
-                    if (this.stream.isSendVideo() || this.stream.isSendAudio()) {
+                    const definedAudioConstraint = ((constraints.audio === undefined) ? true : constraints.audio);
-                        const definedAudioConstraint = ((constraints.audio === undefined) ? true : constraints.audio);
+                    constraintsAux.audio = this.stream.isSendScreen() ? false : definedAudioConstraint;
-                        constraintsAux.audio = this.stream.isSendScreen() ? false : definedAudioConstraint;
+                    constraintsAux.video = constraints.video;
-                        constraintsAux.video = constraints.video;
+                    startTime = Date.now();
-                        startTime = Date.now();
+                    this.setPermissionDialogTimer(timeForDialogEvent);
-                        this.setPermissionDialogTimer(timeForDialogEvent);
 
-                        if (this.stream.isSendScreen() && navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') {
+                    if (this.stream.isSendScreen() && navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') {
 
-                            navigator.mediaDevices['getDisplayMedia']({ video: true })
+                        navigator.mediaDevices['getDisplayMedia']({ video: true })
-                                .then(mediaStream => {
+                            .then(mediaStream => {
-                                    getMediaSuccess(mediaStream, definedAudioConstraint);
+                                getMediaSuccess(mediaStream, definedAudioConstraint);
-                                })
+                            })
-                                .catch(error => {
+                            .catch(error => {
-                                    getMediaError(error);
+                                getMediaError(error);
-                                });
+                            });
 
-                        } else {
-
-                            navigator.mediaDevices.getUserMedia(constraintsAux)
-                                .then(mediaStream => {
-                                    getMediaSuccess(mediaStream, definedAudioConstraint);
-                                })
-                                .catch(error => {
-                                    getMediaError(error);
-                                });
-
-                        }
                     } else {
-                        reject(new OpenViduError(OpenViduErrorName.NO_INPUT_SOURCE_SET,
+
-                            "Properties 'audioSource' and 'videoSource' cannot be set to false or null at the same time when calling 'OpenVidu.initPublisher'"));
+                        navigator.mediaDevices.getUserMedia(constraintsAux)
+                            .then(mediaStream => {
+                                getMediaSuccess(mediaStream, definedAudioConstraint);
+                            })
+                            .catch(error => {
+                                getMediaError(error);
+                            });
                     }
+
                 })
                 .catch((error: OpenViduError) => {
                     errorCallback(error);

openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts

@@ -34,7 +34,8 @@ export enum OpenViduErrorName {
 
     /**
      * The required input device is probably being used by other process when the browser asked for it.
-     * Accuracy of this property is only granted for Chrome and Firefox clients. 
+     * This error can also be triggered when the user granted permission to use the devices but a hardware
+     * error occurred at the OS, browser or web page level, which prevented access to the device.
      * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     DEVICE_ALREADY_IN_USE = "DEVICE_ALREADY_IN_USE",
@@ -47,36 +48,42 @@ export enum OpenViduErrorName {
 
     /**
      * Browser does not support screen sharing.
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     SCREEN_SHARING_NOT_SUPPORTED = 'SCREEN_SHARING_NOT_SUPPORTED',
 
     /**
      * Only for Chrome, there's no screen sharing extension installed
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     SCREEN_EXTENSION_NOT_INSTALLED = 'SCREEN_EXTENSION_NOT_INSTALLED',
 
     /**
      * Only for Chrome, the screen sharing extension is installed but is disabled
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     SCREEN_EXTENSION_DISABLED = 'SCREEN_EXTENSION_DISABLED',
 
     /**
      * No video input device found with the provided deviceId (property [[PublisherProperties.videoSource]])
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     INPUT_VIDEO_DEVICE_NOT_FOUND = 'INPUT_VIDEO_DEVICE_NOT_FOUND',
 
     /**
      * No audio input device found with the provided deviceId (property [[PublisherProperties.audioSource]])
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     INPUT_AUDIO_DEVICE_NOT_FOUND = 'INPUT_AUDIO_DEVICE_NOT_FOUND',
 
     /**
-     * Method [[OpenVidu.initPublisher]] has been called with properties `videoSource` and `audioSource` of
+     * There was an unknown error when trying to access the specified audio device
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
+     */
+    INPUT_AUDIO_DEVICE_GENERIC_ERROR = 'INPUT_AUDIO_DEVICE_GENERIC_ERROR',
+
+    /**
+     * Method [[OpenVidu.initPublisher]] or  [[OpenVidu.getUserMedia]] has been called with properties `videoSource` and `audioSource` of
      * [[PublisherProperties]] parameter both set to *false* or *null*
      */
     NO_INPUT_SOURCE_SET = 'NO_INPUT_SOURCE_SET',
@@ -84,7 +91,7 @@ export enum OpenViduErrorName {
     /**
      * Some media property of [[PublisherProperties]] such as `frameRate` or `resolution` is not supported
      * by the input devices (whenever it is possible they are automatically adjusted to the most similar value).
-     * Returned upon unsuccessful [[OpenVidu.initPublisher]]
+     * Returned upon unsuccessful [[OpenVidu.initPublisher]] or [[OpenVidu.getUserMedia]]
      */
     PUBLISHER_PROPERTIES_ERROR = 'PUBLISHER_PROPERTIES_ERROR',