ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

openvidu-browser: full video dimensions refactoring

pull/621/head
pabloFuente 2021-03-24 19:39:49 +01:00
parent 928514d57c
commit ef013ca5e0
5 changed files with 155 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
openvidu-browser/src/OpenVidu/OpenVidu.ts
View File

@ -116,85 +116,71 @@ export class OpenVidu {
*/
ee = new EventEmitter()
onOrientationChanged(handler): void {
(<any>window).addEventListener('orientationchange', handler);
}
constructor() {
platform = PlatformUtils.getInstance();
this.libraryVersion = packageJson.version;
logger.info("'OpenVidu' initialized");
logger.info("openvidu-browser version: " + this.libraryVersion);
logger.info("OpenVidu initialized");
logger.info('Platform detected: ' + platform.getDescription());
logger.info('openvidu-browser version: ' + this.libraryVersion);
if (platform.isMobileDevice()) {
if (platform.isMobileDevice() || platform.isReactNative()) {
// Listen to orientationchange only on mobile devices
(<any>window).addEventListener('orientationchange', () => {
this.onOrientationChanged(() => {
this.publishers.forEach(publisher => {
if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo && !!publisher.stream.streamManager.videos[0]) {
let attempts = 0;
const oldWidth = publisher.stream.videoDimensions.width;
const oldHeight = publisher.stream.videoDimensions.height;
const getNewVideoDimensions = (): Promise<{ newWidth: number, newHeight: number }> => {
return new Promise((resolve, reject) => {
if (platform.isIonicIos()) {
// iOS Ionic. Limitation: must get new dimensions from an existing video element already inserted into DOM
resolve({
newWidth: publisher.stream.streamManager.videos[0].video.videoWidth,
newHeight: publisher.stream.streamManager.videos[0].video.videoHeight
});
} else {
// Rest of platforms
// New resolution got from different places for Chrome and Firefox. Chrome needs a videoWidth and videoHeight of a videoElement.
// Firefox needs getSettings from the videoTrack
const firefoxSettings = publisher.stream.getMediaStream().getVideoTracks()[0].getSettings();
const newWidth = <number>((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.width : publisher.videoReference.videoWidth);
const newHeight = <number>((platform.isFirefoxBrowser() || platform.isFirefoxMobileBrowser()) ? firefoxSettings.height : publisher.videoReference.videoHeight);
resolve({ newWidth, newHeight });
}
});
};
const repeatUntilChange = setInterval(() => {
getNewVideoDimensions().then(newDimensions => {
sendStreamPropertyChangedEvent(oldWidth, oldHeight, newDimensions.newWidth, newDimensions.newHeight);
});
}, 75);
const sendStreamPropertyChangedEvent = (oldWidth, oldHeight, newWidth, newHeight) => {
attempts++;
if (attempts > 10) {
clearTimeout(repeatUntilChange);
}
if (newWidth !== oldWidth || newHeight !== oldHeight) {
publisher.stream.videoDimensions = {
width: newWidth || 0,
height: newHeight || 0
};
this.sendRequest(
'streamPropertyChanged',
{
streamId: publisher.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(publisher.stream.videoDimensions),
reason: 'deviceRotated'
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]);
publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, 'deviceRotated')]);
this.session.sendVideoData(publisher);
}
});
clearTimeout(repeatUntilChange);
}
};
if (publisher.stream.isLocalStreamPublished && !!publisher.stream && !!publisher.stream.hasVideo) {
this.sendNewVideoDimensionsIfRequired(publisher, 'deviceRotated', 75, 10);
}
});
});
}
}
sendNewVideoDimensionsIfRequired(publisher: Publisher, reason: string, WAIT_INTERVAL: number, MAX_ATTEMPTS: number) {
let attempts = 0;
const oldWidth = publisher.stream.videoDimensions.width;
const oldHeight = publisher.stream.videoDimensions.height;
const repeatUntilChangeOrMaxAttempts: NodeJS.Timeout = setInterval(() => {
attempts++;
if (attempts > MAX_ATTEMPTS) {
clearTimeout(repeatUntilChangeOrMaxAttempts);
}
publisher.getVideoDimensions(publisher.stream.getMediaStream()).then(newDimensions => {
if (newDimensions.width !== oldWidth || newDimensions.height !== oldHeight) {
clearTimeout(repeatUntilChangeOrMaxAttempts);
this.sendVideoDimensionsChangedEvent(publisher, reason, oldWidth, oldHeight, newDimensions.width, newDimensions.height);
}
});
}, WAIT_INTERVAL);
}
sendVideoDimensionsChangedEvent(publisher: Publisher, reason: string, oldWidth: number, oldHeight: number, newWidth: number, newHeight: number) {
publisher.stream.videoDimensions = {
width: newWidth || 0,
height: newHeight || 0
};
this.sendRequest(
'streamPropertyChanged',
{
streamId: publisher.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(publisher.stream.videoDimensions),
reason
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]);
publisher.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(publisher, publisher.stream, 'videoDimensions', publisher.stream.videoDimensions, { width: oldWidth, height: oldHeight }, reason)]);
this.session.sendVideoData(publisher);
}
});
};
/**
* Returns new session

201
openvidu-browser/src/OpenVidu/Publisher.ts
View File

@ -315,7 +315,10 @@ export class Publisher extends StreamManager {
mediaStream.removeTrack(removedTrack);
removedTrack.stop();
mediaStream.addTrack(track);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
if (this.stream.isLocalStreamPublished) {
this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
}
}
return new Promise((resolve, reject) => {
@ -326,17 +329,20 @@ export class Publisher extends StreamManager {
if (track.kind === 'video') {
sender = senders.find(s => !!s.track && s.track.kind === 'video');
if (!sender) {
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'))
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
return;
}
} else if (track.kind === 'audio') {
sender = senders.find(s => !!s.track && s.track.kind === 'audio');
if (!sender) {
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'))
reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
return;
}
} else {
reject(new Error('Unknown track kind ' + track.kind));
return;
}
(<any>sender).replaceTrack(track).then(() => {
(sender as RTCRtpSender).replaceTrack(track).then(() => {
replaceMediaStreamTrack();
resolve();
}).catch(error => {
@ -360,7 +366,7 @@ export class Publisher extends StreamManager {
let constraints: MediaStreamConstraints = {};
let constraintsAux: MediaStreamConstraints = {};
const timeForDialogEvent = 1250;
const timeForDialogEvent = 1500;
let startTime;
const errorCallback = (openViduError: OpenViduError) => {
@ -403,102 +409,38 @@ export class Publisher extends StreamManager {
delete this.firstVideoElement;
if (this.stream.isSendVideo()) {
if (!this.stream.isSendScreen()) {
// Has video track
this.getVideoDimensions(mediaStream).then(dimensions => {
this.stream.videoDimensions = {
width: dimensions.width,
height: dimensions.height
};
if (platform.isIonicIos() || platform.isSafariBrowser()) {
// iOS Ionic or Safari. Limitation: cannot set videoDimensions directly, as the videoReference is not loaded
// if not added to DOM. Must add it to DOM and wait for videoWidth and videoHeight properties to be defined
this.videoReference.style.display = 'none';
document.body.appendChild(this.videoReference);
const videoDimensionsSet = () => {
this.stream.videoDimensions = {
width: this.videoReference.videoWidth,
height: this.videoReference.videoHeight
};
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
document.body.removeChild(this.videoReference);
};
let interval;
this.videoReference.addEventListener('loadedmetadata', () => {
if (this.videoReference.videoWidth === 0) {
interval = setInterval(() => {
if (this.videoReference.videoWidth !== 0) {
clearInterval(interval);
videoDimensionsSet();
}
}, 40);
} else {
videoDimensionsSet();
}
});
} else {
// Rest of platforms
// With no screen share, video dimension can be set directly from MediaStream (getSettings)
// Orientation must be checked for mobile devices (width and height are reversed)
const { width, height } = this.getVideoDimensions(mediaStream);
if (platform.isMobileDevice() && (window.innerHeight > window.innerWidth)) {
// Mobile portrait mode
this.stream.videoDimensions = {
width: height || 0,
height: width || 0
};
} else {
this.stream.videoDimensions = {
width: width || 0,
height: height || 0
};
}
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
}
} else {
// With screen share, video dimension must be got from a video element (onloadedmetadata event)
this.videoReference.addEventListener('loadedmetadata', () => {
this.stream.videoDimensions = {
width: this.videoReference.videoWidth,
height: this.videoReference.videoHeight
};
if (this.stream.isSendScreen()) {
// Set interval to listen for screen resize events
this.screenShareResizeInterval = setInterval(() => {
const firefoxSettings = mediaStream.getVideoTracks()[0].getSettings();
const newWidth = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoWidth : firefoxSettings.width;
const newHeight = (platform.isChromeBrowser() || platform.isOperaBrowser()) ? this.videoReference.videoHeight : firefoxSettings.height;
const settings: MediaTrackSettings = mediaStream.getVideoTracks()[0].getSettings();
const newWidth = settings.width;
const newHeight = settings.height;
if (this.stream.isLocalStreamPublished &&
(newWidth !== this.stream.videoDimensions.width ||
newHeight !== this.stream.videoDimensions.height)) {
const oldValue = { width: this.stream.videoDimensions.width, height: this.stream.videoDimensions.height };
this.stream.videoDimensions = {
width: newWidth || 0,
height: newHeight || 0
};
this.session.openvidu.sendRequest(
'streamPropertyChanged',
{
streamId: this.stream.streamId,
property: 'videoDimensions',
newValue: JSON.stringify(this.stream.videoDimensions),
reason: 'screenResized'
},
(error, response) => {
if (error) {
logger.error("Error sending 'streamPropertyChanged' event", error);
} else {
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]);
this.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this, this.stream, 'videoDimensions', this.stream.videoDimensions, oldValue, 'screenResized')]);
this.session.sendVideoData(this.stream.streamManager);
}
});
(newWidth !== this.stream.videoDimensions.width || newHeight !== this.stream.videoDimensions.height)) {
this.openvidu.sendVideoDimensionsChangedEvent(
this,
'screenResized',
this.stream.videoDimensions.width,
this.stream.videoDimensions.height,
newWidth || 0,
newHeight || 0
);
}
}, 500);
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
});
}
}, 650);
}
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
});
} else {
// Only audio track (no videoDimensions)
this.stream.isLocalStreamReadyToPublish = true;
this.stream.ee.emitEvent('stream-ready-to-publish', []);
}
@ -665,9 +607,66 @@ export class Publisher extends StreamManager {
/**
* @hidden
*
* To obtain the videoDimensions we wait for the video reference to have enough metadata
* and then try to use MediaStreamTrack.getSettingsMethod(). If not available, then we
* use the HTMLVideoElement properties videoWidth and videoHeight
*/
getVideoDimensions(mediaStream: MediaStream): MediaTrackSettings {
return mediaStream.getVideoTracks()[0].getSettings();
getVideoDimensions(mediaStream: MediaStream): Promise<{ width: number, height: number }> {
return new Promise((resolve, reject) => {
// Ionic iOS and Safari iOS supposedly require the video element to actually exist inside the DOM
const requiresDomInsertion: boolean = platform.isIonicIos() || platform.isIOSWithSafari();
let loadedmetadataListener;
const resolveDimensions = () => {
let width: number;
let height: number;
if (typeof this.stream.getMediaStream().getVideoTracks()[0].getSettings === 'function') {
const settings = this.stream.getMediaStream().getVideoTracks()[0].getSettings();
width = settings.width || this.videoReference.videoWidth;
height = settings.height || this.videoReference.videoHeight;
} else {
logger.warn('MediaStreamTrack does not have getSettings method on ' + platform.getDescription());
width = this.videoReference.videoWidth;
height = this.videoReference.videoHeight;
}
if (loadedmetadataListener != null) {
this.videoReference.removeEventListener('loadedmetadata', loadedmetadataListener);
}
if (requiresDomInsertion) {
document.body.removeChild(this.videoReference);
}
resolve({ width, height });
}
if (this.videoReference.readyState >= 1) {
// The video already has metadata available
// No need of loadedmetadata event
resolveDimensions();
} else {
// The video does not have metadata available yet
// Must listen to loadedmetadata event
loadedmetadataListener = () => {
if (!this.videoReference.videoWidth) {
let interval = setInterval(() => {
if (!!this.videoReference.videoWidth) {
clearInterval(interval);
resolveDimensions();
}
}, 40);
} else {
resolveDimensions();
}
};
this.videoReference.addEventListener('loadedmetadata', loadedmetadataListener);
if (requiresDomInsertion) {
document.body.appendChild(this.videoReference);
}
}
});
}
/**
@ -684,17 +683,15 @@ export class Publisher extends StreamManager {
*/
initializeVideoReference(mediaStream: MediaStream) {
this.videoReference = document.createElement('video');
this.videoReference.setAttribute('muted', 'true');
this.videoReference.style.display = 'none';
if (platform.isSafariBrowser()) {
this.videoReference.setAttribute('playsinline', 'true');
}
this.stream.setMediaStream(mediaStream);
if (!!this.firstVideoElement) {
this.createVideoElement(this.firstVideoElement.targetElement, <VideoInsertMode>this.properties.insertMode);
}
this.videoReference.srcObject = mediaStream;
}

2
openvidu-browser/src/OpenVidu/Session.ts
View File

@ -1193,7 +1193,7 @@ export class Session extends EventDispatcher {
platform.isChromeBrowser() || platform.isChromeMobileBrowser() || platform.isOperaBrowser() ||
platform.isOperaMobileBrowser() || platform.isEdgeBrowser() || platform.isEdgeMobileBrowser() || platform.isElectron() ||
(platform.isSafariBrowser() && !platform.isIonicIos()) || platform.isAndroidBrowser() ||
platform.isSamsungBrowser() || platform.isIonicAndroid() || (platform.isIPhoneOrIPad() && platform.isIOSWithSafari())
platform.isSamsungBrowser() || platform.isIonicAndroid() || platform.isIOSWithSafari()
) {
const obtainAndSendVideo = async () => {
const pc = streamManager.stream.getRTCPeerConnection();

2
openvidu-browser/src/OpenViduInternal/Events/StreamPropertyChangedEvent.ts
View File

@ -41,7 +41,7 @@ export class StreamPropertyChangedEvent extends Event {
* Cause of the change on the stream's property:
* - For `videoActive`: `"publishVideo"`
* - For `audioActive`: `"publishAudio"`
* - For `videoDimensions`: `"deviceRotated"` or `"screenResized"`
* - For `videoDimensions`: `"deviceRotated"`, `"screenResized"` or `"trackReplaced"`
* - For `filter`: `"applyFilter"`, `"execFilterMethod"` or `"removeFilter"`
*/
reason: string;

2
openvidu-browser/src/OpenViduInternal/Utils/Platform.ts
View File

@ -118,7 +118,7 @@ export class PlatformUtils {
*/
public isIOSWithSafari(): boolean {
const userAgent = !!platform.ua ? platform.ua : navigator.userAgent;
return (
return this.isIPhoneOrIPad() && (
/\b(\w*Apple\w*)\b/.test(navigator.vendor) &&
/\b(\w*Safari\w*)\b/.test(userAgent) &&
!/\b(\w*CriOS\w*)\b/.test(userAgent) &&