diff --git a/openvidu-browser/src/OpenVidu/OpenVidu.ts b/openvidu-browser/src/OpenVidu/OpenVidu.ts index 7def45a7..f65d351c 100644 --- a/openvidu-browser/src/OpenVidu/OpenVidu.ts +++ b/openvidu-browser/src/OpenVidu/OpenVidu.ts @@ -133,6 +133,10 @@ export class OpenVidu { * @hidden */ isPro: boolean = false; + /** + * @hidden + */ + isEnterprise: boolean = false; /** * @hidden */ diff --git a/openvidu-browser/src/OpenVidu/Publisher.ts b/openvidu-browser/src/OpenVidu/Publisher.ts index 73dc0b71..df1e2fb0 100644 --- a/openvidu-browser/src/OpenVidu/Publisher.ts +++ b/openvidu-browser/src/OpenVidu/Publisher.ts @@ -332,28 +332,6 @@ export class Publisher extends StreamManager { * @returns A Promise (to which you can optionally subscribe to) that is resolved if the track was successfully replaced and rejected with an Error object in other case */ async replaceTrack(track: MediaStreamTrack): Promise { - - const replaceTrackInMediaStream = (): Promise => { - return new Promise((resolve, reject) => { - const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream(); - let removedTrack: MediaStreamTrack; - if (track.kind === 'video') { - removedTrack = mediaStream.getVideoTracks()[0]; - this.stream.lastVideoTrackConstraints = track.getConstraints(); - } else { - removedTrack = mediaStream.getAudioTracks()[0]; - } - mediaStream.removeTrack(removedTrack); - removedTrack.stop(); - mediaStream.addTrack(track); - if (track.kind === 'video' && this.stream.isLocalStreamPublished) { - this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30); - this.session.sendVideoData(this.stream.streamManager, 5, true, 5); - } - return resolve(); - }); - } - // Set field "enabled" of the new track to the previous value const trackOriginalEnabledValue: boolean = track.enabled; if (track.kind === 'video') { @@ -366,10 +344,10 @@ export class Publisher extends StreamManager { // Only if the Publisher has been published is necessary to call native Web API RTCRtpSender.replaceTrack // If it has not been published yet, replacing it on the MediaStream object is enough await this.replaceTrackInRtcRtpSender(track); - return await replaceTrackInMediaStream(); + return await this.replaceTrackInMediaStream(track); } else { // Publisher not published. Simply replace the track on the local MediaStream - return await replaceTrackInMediaStream(); + return await this.replaceTrackInMediaStream(track); } } catch (error) { track.enabled = trackOriginalEnabledValue; @@ -751,6 +729,26 @@ export class Publisher extends StreamManager { this.videoReference.srcObject = mediaStream; } + /** + * @hidden + */ + async replaceTrackInMediaStream(track: MediaStreamTrack): Promise { + const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream(); + let removedTrack: MediaStreamTrack; + if (track.kind === 'video') { + removedTrack = mediaStream.getVideoTracks()[0]; + this.stream.lastVideoTrackConstraints = track.getConstraints(); + } else { + removedTrack = mediaStream.getAudioTracks()[0]; + } + mediaStream.removeTrack(removedTrack); + removedTrack.stop(); + mediaStream.addTrack(track); + if (track.kind === 'video' && this.stream.isLocalStreamPublished) { + this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30); + this.session.sendVideoData(this.stream.streamManager, 5, true, 5); + } + } /* Private methods */ @@ -768,27 +766,23 @@ export class Publisher extends StreamManager { } } - private replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise { - return new Promise((resolve, reject) => { - const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders(); - let sender: RTCRtpSender | undefined; - if (track.kind === 'video') { - sender = senders.find(s => !!s.track && s.track.kind === 'video'); - if (!sender) { - return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')); - } - } else if (track.kind === 'audio') { - sender = senders.find(s => !!s.track && s.track.kind === 'audio'); - if (!sender) { - return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object')); - } - } else { - return reject(new Error('Unknown track kind ' + track.kind)); + private async replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise { + const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders(); + let sender: RTCRtpSender | undefined; + if (track.kind === 'video') { + sender = senders.find(s => !!s.track && s.track.kind === 'video'); + if (!sender) { + throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'); } - (sender as RTCRtpSender).replaceTrack(track) - .then(() => resolve()) - .catch(error => reject(error)); - }); + } else if (track.kind === 'audio') { + sender = senders.find(s => !!s.track && s.track.kind === 'audio'); + if (!sender) { + throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'); + } + } else { + throw new Error('Unknown track kind ' + track.kind); + } + await (sender as RTCRtpSender).replaceTrack(track); } } diff --git a/openvidu-browser/src/OpenVidu/Session.ts b/openvidu-browser/src/OpenVidu/Session.ts index 9702ed03..55770ba9 100644 --- a/openvidu-browser/src/OpenVidu/Session.ts +++ b/openvidu-browser/src/OpenVidu/Session.ts @@ -1486,6 +1486,7 @@ export class Session extends EventDispatcher { const recorder = queryParams['recorder']; const webrtcStatsInterval = queryParams['webrtcStatsInterval']; const sendBrowserLogs = queryParams['sendBrowserLogs']; + const edition = queryParams['edition']; if (!!secret) { this.openvidu.secret = secret; @@ -1500,6 +1501,7 @@ export class Session extends EventDispatcher { this.openvidu.sendBrowserLogs = sendBrowserLogs; } this.openvidu.isPro = !!webrtcStatsInterval && !!sendBrowserLogs; + this.openvidu.isEnterprise = edition === 'enterprise'; this.openvidu.wsUri = 'wss://' + url.host + '/openvidu'; this.openvidu.httpUri = 'https://' + url.host; diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index 4e5e2408..139578e5 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -17,6 +17,7 @@ import { Connection } from './Connection'; import { Filter } from './Filter'; +import { Publisher } from './Publisher'; import { Session } from './Session'; import { StreamManager } from './StreamManager'; import { Subscriber } from './Subscriber'; @@ -33,6 +34,8 @@ import { TypeOfVideo } from '../OpenViduInternal/Enums/TypeOfVideo'; import { OpenViduLogger } from '../OpenViduInternal/Logger/OpenViduLogger'; import { PlatformUtils } from '../OpenViduInternal/Utils/Platform'; +import { v4 as uuidv4 } from 'uuid'; + /** * @hidden */ @@ -151,6 +154,9 @@ export class Stream { private isSubscribeToRemote = false; + private virtualBackgroundSourceElements: { videoClone: HTMLVideoElement, mediaStreamClone: MediaStream }; + private virtualBackgroundSinkElements: { VB: any, video: HTMLVideoElement, canvas: HTMLCanvasElement }; + /** * @hidden */ @@ -308,40 +314,132 @@ export class Stream { * @returns A Promise (to which you can optionally subscribe to) that is resolved to the applied filter if success and rejected with an Error object if not */ applyFilter(type: string, options: Object): Promise { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { - if (!this.session.sessionConnected()) { - return reject(this.session.notConnectedError()); - } - - logger.info('Applying filter to stream ' + this.streamId); - options = options != null ? options : {}; - let optionsString = options; - if (typeof optionsString !== 'string') { - optionsString = JSON.stringify(optionsString); - } - this.session.openvidu.sendRequest( - 'applyFilter', - { streamId: this.streamId, type, options: optionsString }, - (error, response) => { - if (error) { - logger.error('Error applying filter for Stream ' + this.streamId, error); - if (error.code === 401) { - return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to apply a filter")); - } else { - return reject(error); - } + const resolveApplyFilter = (error) => { + if (error) { + logger.error('Error applying filter for Stream ' + this.streamId, error); + if (error.code === 401) { + return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to apply a filter")); } else { - logger.info('Filter successfully applied on Stream ' + this.streamId); - const oldValue: Filter = this.filter!; - this.filter = new Filter(type, options); - this.filter.stream = this; - this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter, oldValue, 'applyFilter')]); - this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter, oldValue, 'applyFilter')]); - return resolve(this.filter); + return reject(error); + } + } else { + logger.info('Filter successfully applied on Stream ' + this.streamId); + const oldValue: Filter = this.filter!; + this.filter = new Filter(type, options); + this.filter.stream = this; + this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter, oldValue, 'applyFilter')]); + this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter, oldValue, 'applyFilter')]); + return resolve(this.filter); + } + } + + if (type === 'VB:blur') { + + // Client filters + + if (!this.session.openvidu.httpUri) { + return reject(this.session.notConnectedError()); + } + if (!this.session.openvidu.isEnterprise) { + return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_EDITION_NOT_SUPPORTED, 'OpenVidu Virtual Background API is part of OpenVidu Enterprise edition')); + } + if (!this.hasVideo) { + return reject(new OpenViduError(OpenViduErrorName.NO_VIDEO_TRACK, 'The Virtual Background filter requires a video track to be applied')); + } + if (!this.mediaStream || this.streamManager.videos.length === 0) { + return reject(new OpenViduError(OpenViduErrorName.STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT, 'The StreamManager requires some video element to be attached to it in order to apply a Virtual Background filter')); + } + + logger.info('Applying client filter to stream ' + this.streamId); + + const afterScriptLoaded = async () => { + try { + const id = this.streamId + '_' + uuidv4(); + const mediaStreamClone = this.mediaStream!.clone(); + const videoClone = this.streamManager.videos[0].video.cloneNode(false) as HTMLVideoElement; + videoClone.id = 'source_video_' + id; + videoClone.srcObject = mediaStreamClone; + this.virtualBackgroundSourceElements = { videoClone, mediaStreamClone }; + + // @ts-ignore + VirtualBackground.VirtualBackground.hideHtmlElement(videoClone, false); + // @ts-ignore + VirtualBackground.VirtualBackground.appendHtmlElementToHiddenContainer(videoClone); + + await videoClone.play(); + + // @ts-ignore + const VB = new VirtualBackground.VirtualBackground({ + id, + openviduServerUrl: new URL(this.session.openvidu.httpUri), + inputVideo: videoClone, + inputResolution: '160x96', + outputFramerate: 30 + }); + const response: { video: HTMLVideoElement, canvas: HTMLCanvasElement } = await VB.backgroundBlur({ + maskRadius: 0.1, + backgroundCoverage: 0.6, + lightWrapping: 0.3 + }); + this.virtualBackgroundSinkElements = { VB, ...response }; + + videoClone.style.display = 'none'; + + if (this.streamManager.remote) { + this.streamManager.replaceTrackInMediaStream((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]); + } else { + (this.streamManager as Publisher).replaceTrack((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]); + } + + resolveApplyFilter(undefined); + + } catch (error) { + resolveApplyFilter(error); } } - ); + + // @ts-ignore + if (typeof VirtualBackground === "undefined") { + let script: HTMLScriptElement = document.createElement("script"); + script.type = "text/javascript"; + script.src = this.session.openvidu.httpUri + '/virtual-background/openvidu-virtual-background.js'; + script.onload = async () => { + await afterScriptLoaded(); + resolve(new Filter(type, options)); + }; + document.body.appendChild(script); + } else { + afterScriptLoaded() + .then(() => resolve(new Filter(type, options))) + .catch(error => reject(error)); + } + + } else { + + // Server filters + + if (!this.session.sessionConnected()) { + return reject(this.session.notConnectedError()); + } + + logger.info('Applying server filter to stream ' + this.streamId); + options = options != null ? options : {}; + let optionsString = options; + if (typeof optionsString !== 'string') { + optionsString = JSON.stringify(optionsString); + } + this.session.openvidu.sendRequest( + 'applyFilter', + { streamId: this.streamId, type, options: optionsString }, + (error, response) => { + resolveApplyFilter(error); + } + ); + + } + }); } @@ -351,34 +449,71 @@ export class Stream { * @returns A Promise (to which you can optionally subscribe to) that is resolved if the previously applied filter has been successfully removed and rejected with an Error object in other case */ removeFilter(): Promise { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { - if (!this.session.sessionConnected()) { - return reject(this.session.notConnectedError()); + const resolveRemoveFilter = (error) => { + if (error) { + logger.error('Error removing filter for Stream ' + this.streamId, error); + if (error.code === 401) { + return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to remove a filter")); + } else { + return reject(error); + } + } else { + logger.info('Filter successfully removed from Stream ' + this.streamId); + const oldValue = this.filter!; + delete this.filter; + this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter!, oldValue, 'applyFilter')]); + this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter!, oldValue, 'applyFilter')]); + return resolve(); + } } - logger.info('Removing filter of stream ' + this.streamId); - this.session.openvidu.sendRequest( - 'removeFilter', - { streamId: this.streamId }, - (error, response) => { - if (error) { - logger.error('Error removing filter for Stream ' + this.streamId, error); - if (error.code === 401) { - return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to remove a filter")); - } else { - return reject(error); - } - } else { - logger.info('Filter successfully removed from Stream ' + this.streamId); - const oldValue = this.filter!; - delete this.filter; - this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter!, oldValue, 'applyFilter')]); - this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter!, oldValue, 'applyFilter')]); - return resolve(); + if (!!this.filter && this.filter?.type.startsWith('VB:')) { + + // Client filters + + try { + + this.virtualBackgroundSinkElements.VB.cleanUp(); + const parent = this.virtualBackgroundSourceElements.videoClone.parentElement; + this.virtualBackgroundSourceElements.videoClone.remove(); + if (parent!.children.length === 0) { + // @ts-ignore + VirtualBackground.VirtualBackground.removeHiddenContainer(); } + + if (this.streamManager.remote) { + await this.streamManager.replaceTrackInMediaStream(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]); + } else { + await (this.streamManager as Publisher).replaceTrack(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]); + } + + return resolveRemoveFilter(undefined); + + } catch (error) { + return resolveRemoveFilter(error); } - ); + + } else { + + // Server filters + + if (!this.session.sessionConnected()) { + return reject(this.session.notConnectedError()); + } + + logger.info('Removing filter of stream ' + this.streamId); + this.session.openvidu.sendRequest( + 'removeFilter', + { streamId: this.streamId }, + (error, response) => { + return resolveRemoveFilter(error); + } + ); + + } + }); } diff --git a/openvidu-browser/src/OpenVidu/StreamManager.ts b/openvidu-browser/src/OpenVidu/StreamManager.ts index b55b9298..e9dd81a0 100644 --- a/openvidu-browser/src/OpenVidu/StreamManager.ts +++ b/openvidu-browser/src/OpenVidu/StreamManager.ts @@ -47,7 +47,7 @@ let platform: PlatformUtils; * * See available event listeners at [[StreamManagerEventMap]]. */ -export class StreamManager extends EventDispatcher { +export abstract class StreamManager extends EventDispatcher { /** * The Stream represented in the DOM by the Publisher/Subscriber @@ -526,6 +526,11 @@ export class StreamManager extends EventDispatcher { this.deactivateStreamPlayingEventExceptionTimeout(); } + /** + * @hidden + */ + abstract replaceTrackInMediaStream(track: MediaStreamTrack): Promise; + /* Private methods */ protected pushNewStreamManagerVideo(streamManagerVideo: StreamManagerVideo) { diff --git a/openvidu-browser/src/OpenVidu/Subscriber.ts b/openvidu-browser/src/OpenVidu/Subscriber.ts index 9e43c9f0..a8af2c2e 100644 --- a/openvidu-browser/src/OpenVidu/Subscriber.ts +++ b/openvidu-browser/src/OpenVidu/Subscriber.ts @@ -73,4 +73,23 @@ export class Subscriber extends StreamManager { return this; } + /* Hidden methods */ + + /** + * @hidden + */ + async replaceTrackInMediaStream(track: MediaStreamTrack): Promise { + const mediaStream: MediaStream = this.stream.getMediaStream(); + let removedTrack: MediaStreamTrack; + if (track.kind === 'video') { + removedTrack = mediaStream.getVideoTracks()[0]; + this.stream.lastVideoTrackConstraints = track.getConstraints(); + } else { + removedTrack = mediaStream.getAudioTracks()[0]; + } + mediaStream.removeTrack(removedTrack); + removedTrack.stop(); + mediaStream.addTrack(track); + } + } \ No newline at end of file diff --git a/openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts b/openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts index cf9ae97e..11fe9334 100644 --- a/openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts +++ b/openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts @@ -107,6 +107,21 @@ export enum OpenViduErrorName { */ OPENVIDU_NOT_CONNECTED = 'OPENVIDU_NOT_CONNECTED', + /** + * The action performed is not supported for this OpenVidu edition. + */ + OPENVIDU_EDITION_NOT_SUPPORTED = 'OPENVIDU_EDITION_NOT_SUPPORTED', + + /** + * The action performed requires a video track to be present. + */ + NO_VIDEO_TRACK = 'NO_VIDEO_TRACK', + + /** + * The action performed requires some video element to be attached to the [[StreamManager]]. + */ + STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT = 'STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT', + /** * Generic error */