From df4db147f59289fbf8dabfb2cfd1b91be4aacbc3 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Thu, 20 Feb 2020 20:14:46 +0100 Subject: [PATCH] openvidu-browser: OpenVidu.getUserMedia fix. generateMediaConstraints refactoring --- openvidu-browser/src/OpenVidu/OpenVidu.ts | 340 +++++++++++------- openvidu-browser/src/OpenVidu/Publisher.ts | 51 +-- .../Private/CustomMediaStreamConstraints.ts | 22 ++ 3 files changed, 253 insertions(+), 160 deletions(-) create mode 100644 openvidu-browser/src/OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints.ts diff --git a/openvidu-browser/src/OpenVidu/OpenVidu.ts b/openvidu-browser/src/OpenVidu/OpenVidu.ts index 78f0b07c..198f5d7e 100644 --- a/openvidu-browser/src/OpenVidu/OpenVidu.ts +++ b/openvidu-browser/src/OpenVidu/OpenVidu.ts @@ -23,6 +23,7 @@ import { StreamPropertyChangedEvent } from '../OpenViduInternal/Events/StreamPro import { Device } from '../OpenViduInternal/Interfaces/Public/Device'; import { OpenViduAdvancedConfiguration } from '../OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration'; import { PublisherProperties } from '../OpenViduInternal/Interfaces/Public/PublisherProperties'; +import { CustomMediaStreamConstraints } from '../OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints'; import { OpenViduError, OpenViduErrorName } from '../OpenViduInternal/Enums/OpenViduError'; import { VideoInsertMode } from '../OpenViduInternal/Enums/VideoInsertMode'; @@ -551,20 +552,41 @@ export class OpenVidu { }); } - this.generateMediaConstraints(options) - .then(constraints => { + this.generateMediaConstraints(options).then(myConstraints => { + + if (!!myConstraints.videoTrack && !!myConstraints.audioTrack || + !!myConstraints.audioTrack && myConstraints.constraints?.video === false || + !!myConstraints.videoTrack && myConstraints.constraints?.audio === false) { + + // No need to call getUserMedia at all. Both tracks provided, or only AUDIO track provided or only VIDEO track provided + resolve(this.addAlreadyProvidedTracks(myConstraints, new MediaStream())); + + } else { + // getUserMedia must be called. AUDIO or VIDEO are requesting a new track + + // Delete already provided constraints for audio or video + if (!!myConstraints.videoTrack) { + delete myConstraints.constraints!.video; + } + if (!!myConstraints.audioTrack) { + delete myConstraints.constraints!.audio; + } + let mustAskForAudioTrackLater = false; if (typeof options.videoSource === 'string') { + // Video is deviceId or screen sharing if (options.videoSource === 'screen' || options.videoSource === 'window' || (platform.name === 'Electron' && options.videoSource.startsWith('screen:'))) { - // Screen sharing - mustAskForAudioTrackLater = options.audioSource !== null && options.audioSource !== false; + // Video is screen sharing + mustAskForAudioTrackLater = !myConstraints.audioTrack && (options.audioSource !== null && options.audioSource !== false); if (navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') { + // getDisplayMedia supported navigator.mediaDevices['getDisplayMedia']({ video: true }) .then(mediaStream => { + this.addAlreadyProvidedTracks(myConstraints, mediaStream); if (mustAskForAudioTrackLater) { - askForAudioStreamOnly(mediaStream, constraints); + askForAudioStreamOnly(mediaStream, myConstraints.constraints); return; } else { resolve(mediaStream); @@ -575,15 +597,21 @@ export class OpenVidu { const errorMessage = error.toString(); reject(new OpenViduError(errorName, errorMessage)); }); + return; + } else { + // getDisplayMedia NOT supported. Can perform getUserMedia below with already calculated constraints } - return; + } else { + // Video is deviceId. Can perform getUserMedia below with already calculated constraints } } - const constraintsAux = mustAskForAudioTrackLater ? { video: constraints.video } : constraints; + // Use already calculated constraints + const constraintsAux = mustAskForAudioTrackLater ? { video: myConstraints.constraints!.video } : myConstraints.constraints; navigator.mediaDevices.getUserMedia(constraintsAux) .then(mediaStream => { + this.addAlreadyProvidedTracks(myConstraints, mediaStream); if (mustAskForAudioTrackLater) { - askForAudioStreamOnly(mediaStream, constraints); + askForAudioStreamOnly(mediaStream, myConstraints.constraints); return; } else { resolve(mediaStream); @@ -599,10 +627,10 @@ export class OpenVidu { } reject(new OpenViduError(errorName, errorMessage)); }); - }) - .catch((error: OpenViduError) => { - reject(error); - }); + } + }).catch((error: OpenViduError) => { + reject(error); + }); }); } @@ -638,168 +666,192 @@ export class OpenVidu { /** * @hidden */ - generateMediaConstraints(publisherProperties: PublisherProperties): Promise { - return new Promise((resolve, reject) => { - let audio, video; + generateMediaConstraints(publisherProperties: PublisherProperties): Promise { + return new Promise((resolve, reject) => { - if (publisherProperties.audioSource === null || publisherProperties.audioSource === false) { - audio = false; - } else if (publisherProperties.audioSource === undefined) { - audio = true; - } else { - audio = publisherProperties.audioSource; + const myConstraints: CustomMediaStreamConstraints = { + audioTrack: undefined, + videoTrack: undefined, + constraints: { + audio: undefined, + video: undefined + } } + const audioSource = publisherProperties.audioSource; + const videoSource = publisherProperties.videoSource; - if (publisherProperties.videoSource === null || publisherProperties.videoSource === false) { - video = false; - } else { - video = { - height: { - ideal: 480 - }, - width: { - ideal: 640 - } - }; + // CASE 1: null/false + if (audioSource === null || audioSource === false) { + // No audio track + myConstraints.constraints!.audio = false; } - - if (audio === false && video === false) { + if (videoSource === null || videoSource === false) { + // No video track + myConstraints.constraints!.video = false; + } + if (myConstraints.constraints!.audio === false && myConstraints.constraints!.video === false) { + // ERROR! audioSource and videoSource cannot be both false at the same time reject(new OpenViduError(OpenViduErrorName.NO_INPUT_SOURCE_SET, "Properties 'audioSource' and 'videoSource' cannot be set to false or null at the same time")); } - const mediaConstraints: MediaStreamConstraints = { - audio, - video - }; - - if (typeof mediaConstraints.audio === 'string') { - mediaConstraints.audio = { deviceId: { exact: mediaConstraints.audio } }; + // CASE 2: MediaStreamTracks + if (typeof MediaStreamTrack !== 'undefined' && audioSource instanceof MediaStreamTrack) { + // Already provided audio track + myConstraints.audioTrack = audioSource; + } + if (typeof MediaStreamTrack !== 'undefined' && videoSource instanceof MediaStreamTrack) { + // Already provided video track + myConstraints.videoTrack = videoSource; } - if (mediaConstraints.video) { + // CASE 3: Default tracks + if (audioSource === undefined) { + myConstraints.constraints!.audio = true; + } + if (videoSource === undefined) { + myConstraints.constraints!.video = { + width: { + ideal: 640 + }, + height: { + ideal: 480 + } + }; + } + // CASE 3.5: give values to resolution and frameRate if video not null/false + if (videoSource !== null && videoSource !== false) { if (!!publisherProperties.resolution) { const widthAndHeight = publisherProperties.resolution.toLowerCase().split('x'); - const width = Number(widthAndHeight[0]); - const height = Number(widthAndHeight[1]); - (mediaConstraints.video as any).width.ideal = width; - (mediaConstraints.video as any).height.ideal = height; + const idealWidth = Number(widthAndHeight[0]); + const idealHeight = Number(widthAndHeight[1]); + myConstraints.constraints!.video = { + width: { + ideal: idealWidth + }, + height: { + ideal: idealHeight + } + } } - if (!!publisherProperties.frameRate) { - (mediaConstraints.video as any).frameRate = { ideal: publisherProperties.frameRate }; + (myConstraints.constraints!.video).frameRate = { ideal: publisherProperties.frameRate }; } + } - if (!!publisherProperties.videoSource && typeof publisherProperties.videoSource === 'string') { + // CASE 4: deviceId or screen sharing + if (typeof audioSource === 'string') { + myConstraints.constraints!.audio = { deviceId: { exact: audioSource } }; + } + if (typeof videoSource === 'string') { - if (publisherProperties.videoSource === 'screen' || - publisherProperties.videoSource === 'window' || - (platform.name === 'Electron' && publisherProperties.videoSource.startsWith('screen:'))) { + if (!this.isScreenShare(videoSource)) { + if (!myConstraints.constraints!.video) { + myConstraints.constraints!.video = {}; + } + (myConstraints.constraints!.video)['deviceId'] = { exact: videoSource }; + } else { - if (!this.checkScreenSharingCapabilities()) { + // Screen sharing - const error = new OpenViduError(OpenViduErrorName.SCREEN_SHARING_NOT_SUPPORTED, 'You can only screen share in desktop Chrome, Firefox, Opera or Electron. Detected client: ' + platform.name); - console.error(error); - reject(error); + if (!this.checkScreenSharingCapabilities()) { + const error = new OpenViduError(OpenViduErrorName.SCREEN_SHARING_NOT_SUPPORTED, 'You can only screen share in desktop Chrome, Firefox, Opera or Electron. Detected client: ' + platform.name); + console.error(error); + reject(error); + } else { + + if (platform.name === 'Electron') { + const prefix = "screen:"; + const videoSourceString: string = videoSource; + const electronScreenId = videoSourceString.substr(videoSourceString.indexOf(prefix) + prefix.length); + (myConstraints.constraints!.video) = { + mandatory: { + chromeMediaSource: 'desktop', + chromeMediaSourceId: electronScreenId + } + }; + resolve(myConstraints); } else { - if (platform.name === 'Electron') { + if (!!this.advancedConfiguration.screenShareChromeExtension && !(platform.name!.indexOf('Firefox') !== -1) && !navigator.mediaDevices['getDisplayMedia']) { - const prefix = "screen:"; - const videoSourceString: string = publisherProperties.videoSource; - const electronScreenId = videoSourceString.substr(videoSourceString.indexOf(prefix) + prefix.length); - (mediaConstraints['video']) = { - mandatory: { - chromeMediaSource: 'desktop', - chromeMediaSourceId: electronScreenId - } - }; - resolve(mediaConstraints); + // Custom screen sharing extension for Chrome (and Opera) and no support for MediaDevices.getDisplayMedia() - } else { - - if (!!this.advancedConfiguration.screenShareChromeExtension && !(platform.name!.indexOf('Firefox') !== -1) && !navigator.mediaDevices['getDisplayMedia']) { - - // Custom screen sharing extension for Chrome (and Opera) and no support for MediaDevices.getDisplayMedia() - - screenSharing.getScreenConstraints((error, screenConstraints) => { - if (!!error || !!screenConstraints.mandatory && screenConstraints.mandatory.chromeMediaSource === 'screen') { - if (error === 'permission-denied' || error === 'PermissionDeniedError') { - const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop'); - console.error(error); - reject(error); - } else { - const extensionId = this.advancedConfiguration.screenShareChromeExtension!.split('/').pop()!!.trim(); - screenSharing.getChromeExtensionStatus(extensionId, (status) => { - if (status === 'installed-disabled') { - const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension'); - console.error(error); - reject(error); - } - if (status === 'not-installed') { - const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, (this.advancedConfiguration.screenShareChromeExtension)); - console.error(error); - reject(error); - } - }); - } + screenSharing.getScreenConstraints((error, screenConstraints) => { + if (!!error || !!screenConstraints.mandatory && screenConstraints.mandatory.chromeMediaSource === 'screen') { + if (error === 'permission-denied' || error === 'PermissionDeniedError') { + const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop'); + console.error(error); + reject(error); } else { - mediaConstraints.video = screenConstraints; - resolve(mediaConstraints); - } - }); - - } else { - - if (navigator.mediaDevices['getDisplayMedia']) { - // getDisplayMedia support (Chrome >= 72, Firefox >= 66) - resolve(mediaConstraints); - } else { - // Default screen sharing extension for Chrome/Opera, or is Firefox < 66 - const firefoxString = platform.name!.indexOf('Firefox') !== -1 ? publisherProperties.videoSource : undefined; - - screenSharingAuto.getScreenId(firefoxString, (error, sourceId, screenConstraints) => { - if (!!error) { - if (error === 'not-installed') { - const extensionUrl = !!this.advancedConfiguration.screenShareChromeExtension ? this.advancedConfiguration.screenShareChromeExtension : - 'https://chrome.google.com/webstore/detail/openvidu-screensharing/lfcgfepafnobdloecchnfaclibenjold'; - const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, extensionUrl); - console.error(error); - reject(error); - } else if (error === 'installed-disabled') { + const extensionId = this.advancedConfiguration.screenShareChromeExtension!.split('/').pop()!!.trim(); + screenSharing.getChromeExtensionStatus(extensionId, status => { + if (status === 'installed-disabled') { const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension'); console.error(error); reject(error); - } else if (error === 'permission-denied') { - const error = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop'); + } + if (status === 'not-installed') { + const error = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, (this.advancedConfiguration.screenShareChromeExtension)); console.error(error); reject(error); } - } else { - mediaConstraints.video = screenConstraints.video; - resolve(mediaConstraints); - } - }); + }); + return; + } + } else { + myConstraints.constraints!.video = screenConstraints; + resolve(myConstraints); } + }); + return; + } else { + + if (navigator.mediaDevices['getDisplayMedia']) { + // getDisplayMedia support (Chrome >= 72, Firefox >= 66) + resolve(myConstraints); + } else { + // Default screen sharing extension for Chrome/Opera, or is Firefox < 66 + const firefoxString = platform.name!.indexOf('Firefox') !== -1 ? publisherProperties.videoSource : undefined; + + screenSharingAuto.getScreenId(firefoxString, (error, sourceId, screenConstraints) => { + if (!!error) { + if (error === 'not-installed') { + const extensionUrl = !!this.advancedConfiguration.screenShareChromeExtension ? this.advancedConfiguration.screenShareChromeExtension : + 'https://chrome.google.com/webstore/detail/openvidu-screensharing/lfcgfepafnobdloecchnfaclibenjold'; + const err = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_NOT_INSTALLED, extensionUrl); + console.error(err); + reject(err); + } else if (error === 'installed-disabled') { + const err = new OpenViduError(OpenViduErrorName.SCREEN_EXTENSION_DISABLED, 'You must enable the screen extension'); + console.error(err); + reject(err); + } else if (error === 'permission-denied') { + const err = new OpenViduError(OpenViduErrorName.SCREEN_CAPTURE_DENIED, 'You must allow access to one window of your desktop'); + console.error(err); + reject(err); + } else { + const err = new OpenViduError(OpenViduErrorName.GENERIC_ERROR, 'Unknown error when accessing screen share'); + console.error(err); + console.error(error); + reject(err); + } + } else { + myConstraints.constraints!.video = screenConstraints.video; + resolve(myConstraints); + } + }); + return; } - - publisherProperties.videoSource = 'screen'; - } } - } else { - // tslint:disable-next-line:no-string-literal - mediaConstraints.video['deviceId'] = { exact: publisherProperties.videoSource }; - resolve(mediaConstraints); } - } else { - resolve(mediaConstraints); } - } else { - resolve(mediaConstraints); } + + resolve(myConstraints); }); } @@ -912,6 +964,18 @@ export class OpenVidu { } } + /** + * @hidden + */ + addAlreadyProvidedTracks(myConstraints: CustomMediaStreamConstraints, mediaStream: MediaStream) { + if (!!myConstraints.videoTrack) { + mediaStream.addTrack(myConstraints.videoTrack); + } + if (!!myConstraints.audioTrack) { + mediaStream.addTrack(myConstraints.audioTrack); + } + return mediaStream; + } /* Private methods */ @@ -961,4 +1025,10 @@ export class OpenVidu { } } + private isScreenShare(videoSource: string) { + return videoSource === 'screen' || + videoSource === 'window' || + (platform.name === 'Electron' && videoSource.startsWith('screen:')) + } + } \ No newline at end of file diff --git a/openvidu-browser/src/OpenVidu/Publisher.ts b/openvidu-browser/src/OpenVidu/Publisher.ts index 84e6ca38..5ea5a40e 100644 --- a/openvidu-browser/src/OpenVidu/Publisher.ts +++ b/openvidu-browser/src/OpenVidu/Publisher.ts @@ -279,7 +279,22 @@ export class Publisher extends StreamManager { */ replaceTrack(track: MediaStreamTrack): Promise { return new Promise((resolve, reject) => { - this.stream.getRTCPeerConnection().getSenders()[0].replaceTrack(track).then(() => { + const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders(); + let sender: RTCRtpSender | undefined; + if (track.kind === 'video') { + sender = senders.find(s => !!s.track && s.track.kind === 'video'); + if (!sender) { + reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) + } + } else if (track.kind === 'audio') { + sender = senders.find(s => !!s.track && s.track.kind === 'audio'); + if (!sender) { + reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) + } + } else { + reject(new Error('Unknown track kind ' + track.kind)); + } + (sender).replaceTrack(track).then(() => { let removedTrack: MediaStreamTrack; if (track.kind === 'video') { removedTrack = this.stream.getMediaStream().getVideoTracks()[0]; @@ -567,30 +582,17 @@ export class Publisher extends StreamManager { } } - // Check if new constraints need to be generated. No constraints needed if - // - video track is given and no audio - // - audio track is given and no video - // - both video and audio tracks are given - if ((typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack && !this.properties.audioSource) - || (typeof MediaStreamTrack !== 'undefined' && this.properties.audioSource instanceof MediaStreamTrack && !this.properties.videoSource) - || (typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack && this.properties.audioSource instanceof MediaStreamTrack)) { - const mediaStream = new MediaStream(); - if (typeof MediaStreamTrack !== 'undefined' && this.properties.videoSource instanceof MediaStreamTrack) { - mediaStream.addTrack((this.properties.videoSource)); - } - if (typeof MediaStreamTrack !== 'undefined' && this.properties.audioSource instanceof MediaStreamTrack) { - mediaStream.addTrack((this.properties.audioSource)); - } - // MediaStreamTracks are handled within callback - just call callback with new MediaStream() and let it handle the sources - successCallback(mediaStream); - // Return as we do not need to process further - return; - } - this.openvidu.generateMediaConstraints(this.properties) .then(myConstraints => { - constraints = myConstraints; + if (myConstraints.constraints === undefined) { + // No need to call getUserMedia at all. MediaStreamTracks already provided + successCallback(this.openvidu.addAlreadyProvidedTracks(myConstraints, new MediaStream())); + // Return as we do not need to process further + return; + } + + constraints = myConstraints.constraints; const outboundStreamOptions = { mediaConstraints: constraints, @@ -605,19 +607,18 @@ export class Publisher extends StreamManager { this.setPermissionDialogTimer(timeForDialogEvent); if (this.stream.isSendScreen() && navigator.mediaDevices['getDisplayMedia'] && platform.name !== 'Electron') { - navigator.mediaDevices['getDisplayMedia']({ video: true }) .then(mediaStream => { + this.openvidu.addAlreadyProvidedTracks(myConstraints, mediaStream); getMediaSuccess(mediaStream, definedAudioConstraint); }) .catch(error => { getMediaError(error); }); - } else { - navigator.mediaDevices.getUserMedia(constraintsAux) .then(mediaStream => { + this.openvidu.addAlreadyProvidedTracks(myConstraints, mediaStream); getMediaSuccess(mediaStream, definedAudioConstraint); }) .catch(error => { diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints.ts new file mode 100644 index 00000000..dcc518f3 --- /dev/null +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/CustomMediaStreamConstraints.ts @@ -0,0 +1,22 @@ +/* + * (C) Copyright 2017-2020 OpenVidu (https://openvidu.io) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export interface CustomMediaStreamConstraints { + constraints: MediaStreamConstraints | undefined; + audioTrack: MediaStreamTrack | undefined; + videoTrack: MediaStreamTrack | undefined; +} \ No newline at end of file