From ef013ca5e07ce966e36fdfcb1a8db3bf60724b01 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Wed, 24 Mar 2021 19:39:49 +0100 Subject: [PATCH] openvidu-browser: full video dimensions refactoring --- openvidu-browser/src/OpenVidu/OpenVidu.ts | 120 +++++------ openvidu-browser/src/OpenVidu/Publisher.ts | 201 +++++++++--------- openvidu-browser/src/OpenVidu/Session.ts | 2 +- .../Events/StreamPropertyChangedEvent.ts | 2 +- .../src/OpenViduInternal/Utils/Platform.ts | 2 +- 5 files changed, 155 insertions(+), 172 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/OpenVidu.ts b/openvidu-browser/src/OpenVidu/OpenVidu.ts index 97adb7c3..100e9cb6 100644 --- a/openvidu-browser/src/OpenVidu/OpenVidu.ts +++ b/openvidu-browser/src/OpenVidu/OpenVidu.ts @@ -116,85 +116,71 @@ export class OpenVidu { */ ee = new EventEmitter() + onOrientationChanged(handler): void { + (window).addEventListener('orientationchange', handler); + } + constructor() { platform = PlatformUtils.getInstance(); this.libraryVersion = packageJson.version; - logger.info("'OpenVidu' initialized"); - logger.info("openvidu-browser version: " + this.libraryVersion); + logger.info("OpenVidu initialized"); + logger.info('Platform detected: ' + platform.getDescription()); + logger.info('openvidu-browser version: ' + this.libraryVersion); - if (platform.isMobileDevice()) { + if (platform.isMobileDevice() || platform.isReactNative()) { // Listen to orientationchange only on mobile devices - (window).addEventListener('orientationchange', () => { + this.onOrientationChanged(() => { this.publishers.forEach(publisher => { - if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo && !!publisher.stream.streamManager.videos[0]) { - - let attempts = 0; - - const oldWidth = publisher.stream.videoDimensions.width; - const oldHeight = publisher.stream.videoDimensions.height; - - const getNewVideoDimensions = (): Promise<{ newWidth: number, newHeight: number }> => { - return new Promise((resolve, reject) => { - if (platform.isIonicIos()) { - // iOS Ionic. Limitation: must get new dimensions from an existing video element already inserted into DOM - resolve({ - newWidth: publisher.stream.streamManager.videos[0].video.videoWidth, - newHeight: publisher.stream.streamManager.videos[0].video.videoHeight - }); - } else { - // Rest of platforms - // New resolution got from different places for Chrome and Firefox. Chrome needs a videoWidth and videoHeight of a videoElement. - // Firefox needs getSettings from the videoTrack - const firefoxSettings = publisher.stream.getMediaStream().getVideoTracks()[0].getSettings(); - const newWidth = ((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.width : publisher.videoReference.videoWidth); - const newHeight = ((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.height : publisher.videoReference.videoHeight); - resolve({ newWidth, newHeight }); - } - }); - }; - - const repeatUntilChange = setInterval(() => { - getNewVideoDimensions().then(newDimensions => { - sendStreamPropertyChangedEvent(oldWidth, oldHeight, newDimensions.newWidth, newDimensions.newHeight); - }); - }, 75); - - const sendStreamPropertyChangedEvent = (oldWidth, oldHeight, newWidth, newHeight) => { - attempts++; - if (attempts > 10) { - clearTimeout(repeatUntilChange); - } - if (newWidth !== oldWidth || newHeight !== oldHeight) { - publisher.stream.videoDimensions = { - width: newWidth || 0, - height: newHeight || 0 - }; - this.sendRequest( - 'streamPropertyChanged', - { - streamId: publisher.stream.streamId, - property: 'videoDimensions', - newValue: JSON.stringify(publisher.stream.videoDimensions), - reason: 'deviceRotated' - }, - (error, response) => { - if (error) { - logger.error("Error sending 'streamPropertyChanged' event", error); - } else { - this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]); - publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]); - this.session.sendVideoData(publisher); - } - }); - clearTimeout(repeatUntilChange); - } - }; + if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo) { + this.sendNewVideoDimensionsIfRequired(publisher, 'deviceRotated', 75, 10); } }); }); } } + sendNewVideoDimensionsIfRequired(publisher: Publisher, reason: string, WAIT_INTERVAL: number, MAX_ATTEMPTS: number) { + let attempts = 0; + const oldWidth = publisher.stream.videoDimensions.width; + const oldHeight = publisher.stream.videoDimensions.height; + + const repeatUntilChangeOrMaxAttempts: NodeJS.Timeout = setInterval(() => { + attempts++; + if (attempts > MAX_ATTEMPTS) { + clearTimeout(repeatUntilChangeOrMaxAttempts); + } + publisher.getVideoDimensions(publisher.stream.getMediaStream()).then(newDimensions => { + if (newDimensions.width !== oldWidth || newDimensions.height !== oldHeight) { + clearTimeout(repeatUntilChangeOrMaxAttempts); + this.sendVideoDimensionsChangedEvent(publisher, reason, oldWidth, oldHeight, newDimensions.width, newDimensions.height); + } + }); + }, WAIT_INTERVAL); + } + + sendVideoDimensionsChangedEvent(publisher: Publisher, reason: string, oldWidth: number, oldHeight: number, newWidth: number, newHeight: number) { + publisher.stream.videoDimensions = { + width: newWidth || 0, + height: newHeight || 0 + }; + this.sendRequest( + 'streamPropertyChanged', + { + streamId: publisher.stream.streamId, + property: 'videoDimensions', + newValue: JSON.stringify(publisher.stream.videoDimensions), + reason + }, + (error, response) => { + if (error) { + logger.error("Error sending 'streamPropertyChanged' event", error); + } else { + this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]); + publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]); + this.session.sendVideoData(publisher); + } + }); + }; /** * Returns new session diff --git a/openvidu-browser/src/OpenVidu/Publisher.ts b/openvidu-browser/src/OpenVidu/Publisher.ts index 22b9f478..6567691e 100644 --- a/openvidu-browser/src/OpenVidu/Publisher.ts +++ b/openvidu-browser/src/OpenVidu/Publisher.ts @@ -315,7 +315,10 @@ export class Publisher extends StreamManager { mediaStream.removeTrack(removedTrack); removedTrack.stop(); mediaStream.addTrack(track); - this.session.sendVideoData(this.stream.streamManager, 5, true, 5); + if (this.stream.isLocalStreamPublished) { + this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30); + this.session.sendVideoData(this.stream.streamManager, 5, true, 5); + } } return new Promise((resolve, reject) => { @@ -326,17 +329,20 @@ export class Publisher extends StreamManager { if (track.kind === 'video') { sender = senders.find(s => !!s.track && s.track.kind === 'video'); if (!sender) { - reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) + reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')); + return; } } else if (track.kind === 'audio') { sender = senders.find(s => !!s.track && s.track.kind === 'audio'); if (!sender) { - reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) + reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')); + return; } } else { reject(new Error('Unknown track kind ' + track.kind)); + return; } - (sender).replaceTrack(track).then(() => { + (sender as RTCRtpSender).replaceTrack(track).then(() => { replaceMediaStreamTrack(); resolve(); }).catch(error => { @@ -360,7 +366,7 @@ export class Publisher extends StreamManager { let constraints: MediaStreamConstraints = {}; let constraintsAux: MediaStreamConstraints = {}; - const timeForDialogEvent = 1250; + const timeForDialogEvent = 1500; let startTime; const errorCallback = (openViduError: OpenViduError) => { @@ -403,102 +409,38 @@ export class Publisher extends StreamManager { delete this.firstVideoElement; if (this.stream.isSendVideo()) { - if (!this.stream.isSendScreen()) { + // Has video track + this.getVideoDimensions(mediaStream).then(dimensions => { + this.stream.videoDimensions = { + width: dimensions.width, + height: dimensions.height + }; - if (platform.isIonicIos() || platform.isSafariBrowser()) { - // iOS Ionic or Safari. Limitation: cannot set videoDimensions directly, as the videoReference is not loaded - // if not added to DOM. Must add it to DOM and wait for videoWidth and videoHeight properties to be defined - - this.videoReference.style.display = 'none'; - document.body.appendChild(this.videoReference); - - const videoDimensionsSet = () => { - this.stream.videoDimensions = { - width: this.videoReference.videoWidth, - height: this.videoReference.videoHeight - }; - this.stream.isLocalStreamReadyToPublish = true; - this.stream.ee.emitEvent('stream-ready-to-publish', []); - document.body.removeChild(this.videoReference); - }; - - let interval; - this.videoReference.addEventListener('loadedmetadata', () => { - if (this.videoReference.videoWidth === 0) { - interval = setInterval(() => { - if (this.videoReference.videoWidth !== 0) { - clearInterval(interval); - videoDimensionsSet(); - } - }, 40); - } else { - videoDimensionsSet(); - } - }); - } else { - // Rest of platforms - // With no screen share, video dimension can be set directly from MediaStream (getSettings) - // Orientation must be checked for mobile devices (width and height are reversed) - const { width, height } = this.getVideoDimensions(mediaStream); - - if (platform.isMobileDevice() && (window.innerHeight > window.innerWidth)) { - // Mobile portrait mode - this.stream.videoDimensions = { - width: height || 0, - height: width || 0 - }; - } else { - this.stream.videoDimensions = { - width: width || 0, - height: height || 0 - }; - } - this.stream.isLocalStreamReadyToPublish = true; - this.stream.ee.emitEvent('stream-ready-to-publish', []); - } - } else { - // With screen share, video dimension must be got from a video element (onloadedmetadata event) - this.videoReference.addEventListener('loadedmetadata', () => { - this.stream.videoDimensions = { - width: this.videoReference.videoWidth, - height: this.videoReference.videoHeight - }; + if (this.stream.isSendScreen()) { + // Set interval to listen for screen resize events this.screenShareResizeInterval = setInterval(() => { - const firefoxSettings = mediaStream.getVideoTracks()[0].getSettings(); - const newWidth = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoWidth : firefoxSettings.width; - const newHeight = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoHeight : firefoxSettings.height; + const settings: MediaTrackSettings = mediaStream.getVideoTracks()[0].getSettings(); + const newWidth = settings.width; + const newHeight = settings.height; if (this.stream.isLocalStreamPublished && - (newWidth !== this.stream.videoDimensions.width || - newHeight !== this.stream.videoDimensions.height)) { - const oldValue = { width: this.stream.videoDimensions.width, height: this.stream.videoDimensions.height }; - this.stream.videoDimensions = { - width: newWidth || 0, - height: newHeight || 0 - }; - this.session.openvidu.sendRequest( - 'streamPropertyChanged', - { - streamId: this.stream.streamId, - property: 'videoDimensions', - newValue: JSON.stringify(this.stream.videoDimensions), - reason: 'screenResized' - }, - (error, response) => { - if (error) { - logger.error("Error sending 'streamPropertyChanged' event", error); - } else { - this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]); - this.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]); - this.session.sendVideoData(this.stream.streamManager); - } - }); + (newWidth !== this.stream.videoDimensions.width || newHeight !== this.stream.videoDimensions.height)) { + this.openvidu.sendVideoDimensionsChangedEvent( + this, + 'screenResized', + this.stream.videoDimensions.width, + this.stream.videoDimensions.height, + newWidth || 0, + newHeight || 0 + ); } - }, 500); - this.stream.isLocalStreamReadyToPublish = true; - this.stream.ee.emitEvent('stream-ready-to-publish', []); - }); - } + }, 650); + } + + this.stream.isLocalStreamReadyToPublish = true; + this.stream.ee.emitEvent('stream-ready-to-publish', []); + }); } else { + // Only audio track (no videoDimensions) this.stream.isLocalStreamReadyToPublish = true; this.stream.ee.emitEvent('stream-ready-to-publish', []); } @@ -665,9 +607,66 @@ export class Publisher extends StreamManager { /** * @hidden + * + * To obtain the videoDimensions we wait for the video reference to have enough metadata + * and then try to use MediaStreamTrack.getSettingsMethod(). If not available, then we + * use the HTMLVideoElement properties videoWidth and videoHeight */ - getVideoDimensions(mediaStream: MediaStream): MediaTrackSettings { - return mediaStream.getVideoTracks()[0].getSettings(); + getVideoDimensions(mediaStream: MediaStream): Promise<{ width: number, height: number }> { + return new Promise((resolve, reject) => { + + // Ionic iOS and Safari iOS supposedly require the video element to actually exist inside the DOM + const requiresDomInsertion: boolean = platform.isIonicIos() || platform.isIOSWithSafari(); + + let loadedmetadataListener; + const resolveDimensions = () => { + let width: number; + let height: number; + if (typeof this.stream.getMediaStream().getVideoTracks()[0].getSettings === 'function') { + const settings = this.stream.getMediaStream().getVideoTracks()[0].getSettings(); + width = settings.width || this.videoReference.videoWidth; + height = settings.height || this.videoReference.videoHeight; + } else { + logger.warn('MediaStreamTrack does not have getSettings method on ' + platform.getDescription()); + width = this.videoReference.videoWidth; + height = this.videoReference.videoHeight; + } + + if (loadedmetadataListener != null) { + this.videoReference.removeEventListener('loadedmetadata', loadedmetadataListener); + } + if (requiresDomInsertion) { + document.body.removeChild(this.videoReference); + } + + resolve({ width, height }); + } + + if (this.videoReference.readyState >= 1) { + // The video already has metadata available + // No need of loadedmetadata event + resolveDimensions(); + } else { + // The video does not have metadata available yet + // Must listen to loadedmetadata event + loadedmetadataListener = () => { + if (!this.videoReference.videoWidth) { + let interval = setInterval(() => { + if (!!this.videoReference.videoWidth) { + clearInterval(interval); + resolveDimensions(); + } + }, 40); + } else { + resolveDimensions(); + } + }; + this.videoReference.addEventListener('loadedmetadata', loadedmetadataListener); + if (requiresDomInsertion) { + document.body.appendChild(this.videoReference); + } + } + }); } /** @@ -684,17 +683,15 @@ export class Publisher extends StreamManager { */ initializeVideoReference(mediaStream: MediaStream) { this.videoReference = document.createElement('video'); - + this.videoReference.setAttribute('muted', 'true'); + this.videoReference.style.display = 'none'; if (platform.isSafariBrowser()) { this.videoReference.setAttribute('playsinline', 'true'); } - this.stream.setMediaStream(mediaStream); - if (!!this.firstVideoElement) { this.createVideoElement(this.firstVideoElement.targetElement, this.properties.insertMode); } - this.videoReference.srcObject = mediaStream; } diff --git a/openvidu-browser/src/OpenVidu/Session.ts b/openvidu-browser/src/OpenVidu/Session.ts index 3f60be90..5c9a165a 100644 --- a/openvidu-browser/src/OpenVidu/Session.ts +++ b/openvidu-browser/src/OpenVidu/Session.ts @@ -1193,7 +1193,7 @@ export class Session extends EventDispatcher { platform.isChromeBrowser() || platform.isChromeMobileBrowser() || platform.isOperaBrowser() || platform.isOperaMobileBrowser() || platform.isEdgeBrowser() || platform.isEdgeMobileBrowser() || platform.isElectron() || (platform.isSafariBrowser() && !platform.isIonicIos()) || platform.isAndroidBrowser() || - platform.isSamsungBrowser() || platform.isIonicAndroid() || (platform.isIPhoneOrIPad() && platform.isIOSWithSafari()) + platform.isSamsungBrowser() || platform.isIonicAndroid() || platform.isIOSWithSafari() ) { const obtainAndSendVideo = async () => { const pc = streamManager.stream.getRTCPeerConnection(); diff --git a/openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts b/openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts index ce98cd8e..65745423 100644 --- a/openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts +++ b/openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts @@ -41,7 +41,7 @@ export class StreamPropertyChangedEvent extends Event { * Cause of the change on the stream's property: * - For `videoActive`: `"publishVideo"` * - For `audioActive`: `"publishAudio"` - * - For `videoDimensions`: `"deviceRotated"` or `"screenResized"` + * - For `videoDimensions`: `"deviceRotated"`, `"screenResized"` or `"trackReplaced"` * - For `filter`: `"applyFilter"`, `"execFilterMethod"` or `"removeFilter"` */ reason: string; diff --git a/openvidu-browser/src/OpenViduInternal/Utils/Platform.ts b/openvidu-browser/src/OpenViduInternal/Utils/Platform.ts index 76afa098..f7450dbd 100644 --- a/openvidu-browser/src/OpenViduInternal/Utils/Platform.ts +++ b/openvidu-browser/src/OpenViduInternal/Utils/Platform.ts @@ -118,7 +118,7 @@ export class PlatformUtils { */ public isIOSWithSafari(): boolean { const userAgent = !!platform.ua ? platform.ua : navigator.userAgent; - return ( + return this.isIPhoneOrIPad() && ( /\b(\w*Apple\w*)\b/.test(navigator.vendor) && /\b(\w*Safari\w*)\b/.test(userAgent) && !/\b(\w*CriOS\w*)\b/.test(userAgent) &&