ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

openvidu-browser: full video dimensions refactoring

pull/621/head
pabloFuente 2021-03-24 19:39:49 +01:00
parent 928514d57c
commit ef013ca5e0
5 changed files with 155 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
openvidu-browser/src/OpenVidu/OpenVidu.ts
View File

@ -116,85 +116,71 @@ export class OpenVidu {
*/ */
ee = new EventEmitter() ee = new EventEmitter()
onOrientationChanged(handler): void {
(<any>window).addEventListener('orientationchange', handler);
}
constructor() { constructor() {
platform = PlatformUtils.getInstance(); platform = PlatformUtils.getInstance();
this.libraryVersion = packageJson.version; this.libraryVersion = packageJson.version;
logger.info("'OpenVidu' initialized"); logger.info("OpenVidu initialized");
logger.info("openvidu-browser version: " + this.libraryVersion); logger.info('Platform detected: ' + platform.getDescription());
logger.info('openvidu-browser version: ' + this.libraryVersion);
if (platform.isMobileDevice()) { if (platform.isMobileDevice() || platform.isReactNative()) {
// Listen to orientationchange only on mobile devices // Listen to orientationchange only on mobile devices
(<any>window).addEventListener('orientationchange', () => { this.onOrientationChanged(() => {
this.publishers.forEach(publisher => { this.publishers.forEach(publisher => {
if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo && !!publisher.stream.streamManager.videos[0]) { if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo) {
this.sendNewVideoDimensionsIfRequired(publisher, 'deviceRotated', 75, 10);
let attempts = 0;
const oldWidth = publisher.stream.videoDimensions.width;
const oldHeight = publisher.stream.videoDimensions.height;
const getNewVideoDimensions = (): Promise<{ newWidth: number, newHeight: number }> => {
return new Promise((resolve, reject) => {
if (platform.isIonicIos()) {
// iOS Ionic. Limitation: must get new dimensions from an existing video element already inserted into DOM
resolve({
newWidth: publisher.stream.streamManager.videos[0].video.videoWidth,
newHeight: publisher.stream.streamManager.videos[0].video.videoHeight
});
} else {
// Rest of platforms
// New resolution got from different places for Chrome and Firefox. Chrome needs a videoWidth and videoHeight of a videoElement.
// Firefox needs getSettings from the videoTrack
const firefoxSettings = publisher.stream.getMediaStream().getVideoTracks()[0].getSettings();
const newWidth = <number>((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.width : publisher.videoReference.videoWidth);
const newHeight = <number>((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.height : publisher.videoReference.videoHeight);
resolve({ newWidth, newHeight });
}
});
};
const repeatUntilChange = setInterval(() => {
getNewVideoDimensions().then(newDimensions => {
sendStreamPropertyChangedEvent(oldWidth, oldHeight, newDimensions.newWidth, newDimensions.newHeight);
});
}, 75);
const sendStreamPropertyChangedEvent = (oldWidth, oldHeight, newWidth, newHeight) => {
attempts++;
if (attempts > 10) {
clearTimeout(repeatUntilChange);
}
if (newWidth !== oldWidth || newHeight !== oldHeight) {
publisher.stream.videoDimensions = {
width: newWidth || 0,
height: newHeight || 0
};
this.sendRequest(
'streamPropertyChanged',
{
streamId: publisher.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(publisher.stream.videoDimensions),
reason: 'deviceRotated'
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]);
publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]);
this.session.sendVideoData(publisher);
}
});
clearTimeout(repeatUntilChange);
}
};
} }
}); });
}); });
} }
} }
sendNewVideoDimensionsIfRequired(publisher: Publisher, reason: string, WAIT_INTERVAL: number, MAX_ATTEMPTS: number) {
let attempts = 0;
const oldWidth = publisher.stream.videoDimensions.width;
const oldHeight = publisher.stream.videoDimensions.height;
const repeatUntilChangeOrMaxAttempts: NodeJS.Timeout = setInterval(() => {
attempts++;
if (attempts > MAX_ATTEMPTS) {
clearTimeout(repeatUntilChangeOrMaxAttempts);
}
publisher.getVideoDimensions(publisher.stream.getMediaStream()).then(newDimensions => {
if (newDimensions.width !== oldWidth || newDimensions.height !== oldHeight) {
clearTimeout(repeatUntilChangeOrMaxAttempts);
this.sendVideoDimensionsChangedEvent(publisher, reason, oldWidth, oldHeight, newDimensions.width, newDimensions.height);
}
});
}, WAIT_INTERVAL);
}
sendVideoDimensionsChangedEvent(publisher: Publisher, reason: string, oldWidth: number, oldHeight: number, newWidth: number, newHeight: number) {
publisher.stream.videoDimensions = {
width: newWidth || 0,
height: newHeight || 0
};
this.sendRequest(
'streamPropertyChanged',
{
streamId: publisher.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(publisher.stream.videoDimensions),
reason
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]);
publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]);
this.session.sendVideoData(publisher);
}
});
};
/** /**
* Returns new session * Returns new session

201
openvidu-browser/src/OpenVidu/Publisher.ts
View File

@ -315,7 +315,10 @@ export class Publisher extends StreamManager {
mediaStream.removeTrack(removedTrack); mediaStream.removeTrack(removedTrack);
removedTrack.stop(); removedTrack.stop();
mediaStream.addTrack(track); mediaStream.addTrack(track);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5); if (this.stream.isLocalStreamPublished) {
this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
}
} }
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -326,17 +329,20 @@ export class Publisher extends StreamManager {
if (track.kind === 'video') { if (track.kind === 'video') {
sender = senders.find(s => !!s.track && s.track.kind === 'video'); sender = senders.find(s => !!s.track && s.track.kind === 'video');
if (!sender) { if (!sender) {
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
return;
} }
} else if (track.kind === 'audio') { } else if (track.kind === 'audio') {
sender = senders.find(s => !!s.track && s.track.kind === 'audio'); sender = senders.find(s => !!s.track && s.track.kind === 'audio');
if (!sender) { if (!sender) {
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')) reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
return;
} }
} else { } else {
reject(new Error('Unknown track kind ' + track.kind)); reject(new Error('Unknown track kind ' + track.kind));
return;
} }
(<any>sender).replaceTrack(track).then(() => { (sender as RTCRtpSender).replaceTrack(track).then(() => {
replaceMediaStreamTrack(); replaceMediaStreamTrack();
resolve(); resolve();
}).catch(error => { }).catch(error => {
@ -360,7 +366,7 @@ export class Publisher extends StreamManager {
let constraints: MediaStreamConstraints = {}; let constraints: MediaStreamConstraints = {};
let constraintsAux: MediaStreamConstraints = {}; let constraintsAux: MediaStreamConstraints = {};
const timeForDialogEvent = 1250; const timeForDialogEvent = 1500;
let startTime; let startTime;
const errorCallback = (openViduError: OpenViduError) => { const errorCallback = (openViduError: OpenViduError) => {
@ -403,102 +409,38 @@ export class Publisher extends StreamManager {
delete this.firstVideoElement; delete this.firstVideoElement;
if (this.stream.isSendVideo()) { if (this.stream.isSendVideo()) {
if (!this.stream.isSendScreen()) { // Has video track
this.getVideoDimensions(mediaStream).then(dimensions => {
this.stream.videoDimensions = {
width: dimensions.width,
height: dimensions.height
};
if (platform.isIonicIos() || platform.isSafariBrowser()) { if (this.stream.isSendScreen()) {
// iOS Ionic or Safari. Limitation: cannot set videoDimensions directly, as the videoReference is not loaded // Set interval to listen for screen resize events
// if not added to DOM. Must add it to DOM and wait for videoWidth and videoHeight properties to be defined
this.videoReference.style.display = 'none';
document.body.appendChild(this.videoReference);
const videoDimensionsSet = () => {
this.stream.videoDimensions = {
width: this.videoReference.videoWidth,
height: this.videoReference.videoHeight
};
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
document.body.removeChild(this.videoReference);
};
let interval;
this.videoReference.addEventListener('loadedmetadata', () => {
if (this.videoReference.videoWidth === 0) {
interval = setInterval(() => {
if (this.videoReference.videoWidth !== 0) {
clearInterval(interval);
videoDimensionsSet();
}
}, 40);
} else {
videoDimensionsSet();
}
});
} else {
// Rest of platforms
// With no screen share, video dimension can be set directly from MediaStream (getSettings)
// Orientation must be checked for mobile devices (width and height are reversed)
const { width, height } = this.getVideoDimensions(mediaStream);
if (platform.isMobileDevice() && (window.innerHeight > window.innerWidth)) {
// Mobile portrait mode
this.stream.videoDimensions = {
width: height || 0,
height: width || 0
};
} else {
this.stream.videoDimensions = {
width: width || 0,
height: height || 0
};
}
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
}
} else {
// With screen share, video dimension must be got from a video element (onloadedmetadata event)
this.videoReference.addEventListener('loadedmetadata', () => {
this.stream.videoDimensions = {
width: this.videoReference.videoWidth,
height: this.videoReference.videoHeight
};
this.screenShareResizeInterval = setInterval(() => { this.screenShareResizeInterval = setInterval(() => {
const firefoxSettings = mediaStream.getVideoTracks()[0].getSettings(); const settings: MediaTrackSettings = mediaStream.getVideoTracks()[0].getSettings();
const newWidth = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoWidth : firefoxSettings.width; const newWidth = settings.width;
const newHeight = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoHeight : firefoxSettings.height; const newHeight = settings.height;
if (this.stream.isLocalStreamPublished && if (this.stream.isLocalStreamPublished &&
(newWidth !== this.stream.videoDimensions.width || (newWidth !== this.stream.videoDimensions.width || newHeight !== this.stream.videoDimensions.height)) {
newHeight !== this.stream.videoDimensions.height)) { this.openvidu.sendVideoDimensionsChangedEvent(
const oldValue = { width: this.stream.videoDimensions.width, height: this.stream.videoDimensions.height }; this,
this.stream.videoDimensions = { 'screenResized',
width: newWidth || 0, this.stream.videoDimensions.width,
height: newHeight || 0 this.stream.videoDimensions.height,
}; newWidth || 0,
this.session.openvidu.sendRequest( newHeight || 0
'streamPropertyChanged', );
{
streamId: this.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(this.stream.videoDimensions),
reason: 'screenResized'
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]);
this.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]);
this.session.sendVideoData(this.stream.streamManager);
}
});
} }
}, 500); }, 650);
this.stream.isLocalStreamReadyToPublish = true; }
this.stream.ee.emitEvent('stream-ready-to-publish', []);
}); this.stream.isLocalStreamReadyToPublish = true;
} this.stream.ee.emitEvent('stream-ready-to-publish', []);
});
} else { } else {
// Only audio track (no videoDimensions)
this.stream.isLocalStreamReadyToPublish = true; this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []); this.stream.ee.emitEvent('stream-ready-to-publish', []);
} }
@ -665,9 +607,66 @@ export class Publisher extends StreamManager {
/** /**
* @hidden * @hidden
*
* To obtain the videoDimensions we wait for the video reference to have enough metadata
* and then try to use MediaStreamTrack.getSettingsMethod(). If not available, then we
* use the HTMLVideoElement properties videoWidth and videoHeight
*/ */
getVideoDimensions(mediaStream: MediaStream): MediaTrackSettings { getVideoDimensions(mediaStream: MediaStream): Promise<{ width: number, height: number }> {
return mediaStream.getVideoTracks()[0].getSettings(); return new Promise((resolve, reject) => {
// Ionic iOS and Safari iOS supposedly require the video element to actually exist inside the DOM
const requiresDomInsertion: boolean = platform.isIonicIos() || platform.isIOSWithSafari();
let loadedmetadataListener;
const resolveDimensions = () => {
let width: number;
let height: number;
if (typeof this.stream.getMediaStream().getVideoTracks()[0].getSettings === 'function') {
const settings = this.stream.getMediaStream().getVideoTracks()[0].getSettings();
width = settings.width || this.videoReference.videoWidth;
height = settings.height || this.videoReference.videoHeight;
} else {
logger.warn('MediaStreamTrack does not have getSettings method on ' + platform.getDescription());
width = this.videoReference.videoWidth;
height = this.videoReference.videoHeight;
}
if (loadedmetadataListener != null) {
this.videoReference.removeEventListener('loadedmetadata', loadedmetadataListener);
}
if (requiresDomInsertion) {
document.body.removeChild(this.videoReference);
}
resolve({ width, height });
}
if (this.videoReference.readyState >= 1) {
// The video already has metadata available
// No need of loadedmetadata event
resolveDimensions();
} else {
// The video does not have metadata available yet
// Must listen to loadedmetadata event
loadedmetadataListener = () => {
if (!this.videoReference.videoWidth) {
let interval = setInterval(() => {
if (!!this.videoReference.videoWidth) {
clearInterval(interval);
resolveDimensions();
}
}, 40);
} else {
resolveDimensions();
}
};
this.videoReference.addEventListener('loadedmetadata', loadedmetadataListener);
if (requiresDomInsertion) {
document.body.appendChild(this.videoReference);
}
}
});
} }
/** /**
@ -684,17 +683,15 @@ export class Publisher extends StreamManager {
*/ */
initializeVideoReference(mediaStream: MediaStream) { initializeVideoReference(mediaStream: MediaStream) {
this.videoReference = document.createElement('video'); this.videoReference = document.createElement('video');
this.videoReference.setAttribute('muted', 'true');
this.videoReference.style.display = 'none';
if (platform.isSafariBrowser()) { if (platform.isSafariBrowser()) {
this.videoReference.setAttribute('playsinline', 'true'); this.videoReference.setAttribute('playsinline', 'true');
} }
this.stream.setMediaStream(mediaStream); this.stream.setMediaStream(mediaStream);
if (!!this.firstVideoElement) { if (!!this.firstVideoElement) {
this.createVideoElement(this.firstVideoElement.targetElement, <VideoInsertMode>this.properties.insertMode); this.createVideoElement(this.firstVideoElement.targetElement, <VideoInsertMode>this.properties.insertMode);
} }
this.videoReference.srcObject = mediaStream; this.videoReference.srcObject = mediaStream;
} }

2
openvidu-browser/src/OpenVidu/Session.ts
View File

@ -1193,7 +1193,7 @@ export class Session extends EventDispatcher {
platform.isChromeBrowser() || platform.isChromeMobileBrowser() || platform.isOperaBrowser() || platform.isChromeBrowser() || platform.isChromeMobileBrowser() || platform.isOperaBrowser() ||
platform.isOperaMobileBrowser() || platform.isEdgeBrowser() || platform.isEdgeMobileBrowser() || platform.isElectron() || platform.isOperaMobileBrowser() || platform.isEdgeBrowser() || platform.isEdgeMobileBrowser() || platform.isElectron() ||
(platform.isSafariBrowser() && !platform.isIonicIos()) || platform.isAndroidBrowser() || (platform.isSafariBrowser() && !platform.isIonicIos()) || platform.isAndroidBrowser() ||
platform.isSamsungBrowser() || platform.isIonicAndroid() || (platform.isIPhoneOrIPad() && platform.isIOSWithSafari()) platform.isSamsungBrowser() || platform.isIonicAndroid() || platform.isIOSWithSafari()
) { ) {
const obtainAndSendVideo = async () => { const obtainAndSendVideo = async () => {
const pc = streamManager.stream.getRTCPeerConnection(); const pc = streamManager.stream.getRTCPeerConnection();

2
openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts
View File

@ -41,7 +41,7 @@ export class StreamPropertyChangedEvent extends Event {
* Cause of the change on the stream's property: * Cause of the change on the stream's property:
* - For `videoActive`: `"publishVideo"` * - For `videoActive`: `"publishVideo"`
* - For `audioActive`: `"publishAudio"` * - For `audioActive`: `"publishAudio"`
* - For `videoDimensions`: `"deviceRotated"` or `"screenResized"` * - For `videoDimensions`: `"deviceRotated"`, `"screenResized"` or `"trackReplaced"`
* - For `filter`: `"applyFilter"`, `"execFilterMethod"` or `"removeFilter"` * - For `filter`: `"applyFilter"`, `"execFilterMethod"` or `"removeFilter"`
*/ */
reason: string; reason: string;

2
openvidu-browser/src/OpenViduInternal/Utils/Platform.ts
View File

@ -118,7 +118,7 @@ export class PlatformUtils {
*/ */
public isIOSWithSafari(): boolean { public isIOSWithSafari(): boolean {
const userAgent = !!platform.ua ? platform.ua : navigator.userAgent; const userAgent = !!platform.ua ? platform.ua : navigator.userAgent;
return ( return this.isIPhoneOrIPad() && (
/\b(\w*Apple\w*)\b/.test(navigator.vendor) && /\b(\w*Apple\w*)\b/.test(navigator.vendor) &&
/\b(\w*Safari\w*)\b/.test(userAgent) && /\b(\w*Safari\w*)\b/.test(userAgent) &&
!/\b(\w*CriOS\w*)\b/.test(userAgent) && !/\b(\w*CriOS\w*)\b/.test(userAgent) &&